From 2b0af5bdb41d912757d5cbd000ed4b70437f89e3 Mon Sep 17 00:00:00 2001 From: AndrewVolosytnykhThingsboard Date: Fri, 28 May 2021 15:34:53 +0300 Subject: [PATCH 001/262] Rate limits moved to tenant profile --- .../config/RateLimitProcessingFilter.java | 55 +- .../controller/plugin/TbWebSocketHandler.java | 67 +- .../DefaultTelemetryWebSocketService.java | 78 +-- .../server/common/data/TenantProfile.java | 7 + .../DefaultTenantProfileConfiguration.java | 17 + .../nosql/CassandraBufferedRateExecutor.java | 34 +- .../util/AbstractBufferedRateExecutor.java | 26 +- .../tenant-profile-data.component.html | 30 +- ...enant-profile-configuration.component.html | 584 +++++++++++------- ...-tenant-profile-configuration.component.ts | 31 +- ui-ngx/src/app/shared/models/tenant.model.ts | 38 +- .../assets/locale/locale.constant-en_US.json | 18 +- 12 files changed, 634 insertions(+), 351 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index d9b7573d5e..381e767cce 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -16,16 +16,17 @@ package org.thingsboard.server.config; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import org.springframework.web.filter.GenericFilterBean; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,18 +42,21 @@ import java.util.concurrent.ConcurrentMap; @Component public class RateLimitProcessingFilter extends GenericFilterBean { - @Value("${server.rest.limits.tenant.enabled:false}") - private boolean perTenantLimitsEnabled; - @Value("${server.rest.limits.tenant.configuration:}") - private String perTenantLimitsConfiguration; - @Value("${server.rest.limits.customer.enabled:false}") - private boolean perCustomerLimitsEnabled; - @Value("${server.rest.limits.customer.configuration:}") - private String perCustomerLimitsConfiguration; +// @Value("${server.rest.limits.tenant.enabled:false}") +// private boolean perTenantLimitsEnabled; +// @Value("${server.rest.limits.tenant.configuration:}") +// private String perTenantLimitsConfiguration; +// @Value("${server.rest.limits.customer.enabled:false}") +// private boolean perCustomerLimitsEnabled; +// @Value("${server.rest.limits.customer.configuration:}") +// private String perCustomerLimitsConfiguration; @Autowired private ThingsboardErrorResponseHandler errorResponseHandler; + @Autowired + private TbTenantProfileCache tenantProfileCache; + private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private ConcurrentMap perCustomerLimits = new ConcurrentHashMap<>(); @@ -60,18 +64,29 @@ public class RateLimitProcessingFilter extends GenericFilterBean { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - if (perTenantLimitsEnabled) { - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(user.getTenantId(), id -> new TbRateLimits(perTenantLimitsConfiguration)); - if (!rateLimits.tryConsume()) { - errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response); - return; + + var profile= tenantProfileCache.get(user.getTenantId()).getDefaultTenantConfiguration(); + + if(profile != null) { + if (StringUtils.isNotEmpty(profile.getRateLimitsTenantConfiguration())) { + + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent( + user.getTenantId(), id -> new TbRateLimits(profile.getRateLimitsTenantConfiguration()) + ); + if (!rateLimits.tryConsume()) { + errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response); + return; + } } - } - if (perCustomerLimitsEnabled && user.isCustomerUser()) { - TbRateLimits rateLimits = perCustomerLimits.computeIfAbsent(user.getCustomerId(), id -> new TbRateLimits(perCustomerLimitsConfiguration)); - if (!rateLimits.tryConsume()) { - errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response); - return; + if (StringUtils.isNotEmpty(profile.getRateLimitsCustomerConfiguration())) { + + TbRateLimits rateLimits = perCustomerLimits.computeIfAbsent( + user.getCustomerId(), id -> new TbRateLimits(profile.getRateLimitsCustomerConfiguration()) + ); + if (!rateLimits.tryConsume()) { + errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response); + return; + } } } } diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 0e59030fe2..ea1b02eb7d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -68,22 +69,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Autowired private TelemetryWebSocketService webSocketService; + @Autowired + private TbTenantProfileCache tenantProfileCache; + @Value("${server.ws.send_timeout:5000}") private long sendTimeout; - @Value("${server.ws.limits.max_sessions_per_tenant:0}") - private int maxSessionsPerTenant; - @Value("${server.ws.limits.max_sessions_per_customer:0}") - private int maxSessionsPerCustomer; - @Value("${server.ws.limits.max_sessions_per_regular_user:0}") - private int maxSessionsPerRegularUser; - @Value("${server.ws.limits.max_sessions_per_public_user:0}") - private int maxSessionsPerPublicUser; - @Value("${server.ws.limits.max_queue_per_ws_session:1000}") - private int maxMsgQueuePerSession; - - @Value("${server.ws.limits.max_updates_per_session:}") - private String perSessionUpdatesConfiguration; - @Value("${server.ws.ping_timeout:30000}") private long pingTimeout; @@ -124,10 +114,15 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr String internalSessionId = session.getId(); TelemetryWebSocketSessionRef sessionRef = toRef(session); String externalSessionId = sessionRef.getSessionId(); + if (!checkLimits(session, sessionRef)) { return; } - internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, maxMsgQueuePerSession)); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + if(tenantProfileConfiguration.getWsLimitQueuePerWsSession() <= 0) { + tenantProfileConfiguration.setWsLimitQueuePerWsSession(500); + } + internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsLimitQueuePerWsSession())); externalSessionMap.put(externalSessionId, internalSessionId); processInWebSocketService(sessionRef, SessionEvent.onEstablished()); @@ -287,11 +282,14 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr String externalId = sessionRef.getSessionId(); log.debug("[{}] Processing {}", externalId, msg); String internalId = externalSessionMap.get(externalId); + + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + if (internalId != null) { SessionMetaData sessionMd = internalSessionMap.get(internalId); if (sessionMd != null) { - if (!StringUtils.isEmpty(perSessionUpdatesConfiguration)) { - TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(perSessionUpdatesConfiguration)); + if (!StringUtils.isEmpty(tenantProfileConfiguration.getWsLimitUpdatesPerSession())) { + TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(tenantProfileConfiguration.getWsLimitUpdatesPerSession())); if (!rateLimits.tryConsume()) { if (blacklistedSessions.putIfAbsent(externalId, sessionRef) == null) { log.info("[{}][{}][{}] Failed to process session update. Max session updates limit reached" @@ -347,11 +345,18 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } private boolean checkLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) throws Exception { + var tenantProfileConfiguration = + tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + + if(tenantProfileConfiguration == null) { + return true; + } + String sessionId = session.getId(); - if (maxSessionsPerTenant > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant() > 0) { Set tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSessions) { - if (tenantSessions.size() < maxSessionsPerTenant) { + if (tenantSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant()) { tenantSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max tenant sessions limit reached" @@ -363,10 +368,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (maxSessionsPerCustomer > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer() > 0) { Set customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { - if (customerSessions.size() < maxSessionsPerCustomer) { + if (customerSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer()) { customerSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max customer sessions limit reached" @@ -376,10 +381,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } } } - if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser() > 0 + && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { - if (regularUserSessions.size() < maxSessionsPerRegularUser) { + if (regularUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser()) { regularUserSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max regular user sessions limit reached" @@ -389,10 +395,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } } } - if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser() > 0 + && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { - if (publicUserSessions.size() < maxSessionsPerPublicUser) { + if (publicUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser()) { publicUserSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max public user sessions limit reached" @@ -407,29 +414,31 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } private void cleanupLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) { + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + String sessionId = session.getId(); perSessionUpdateLimits.remove(sessionRef.getSessionId()); blacklistedSessions.remove(sessionRef.getSessionId()); - if (maxSessionsPerTenant > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant() > 0) { Set tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSessions) { tenantSessions.remove(sessionId); } } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (maxSessionsPerCustomer > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer() > 0) { Set customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { customerSessions.remove(sessionId); } } - if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { regularUserSessions.remove(sessionId); } } - if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { publicUserSessions.remove(sessionId); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 73eed820ab..edb36c11c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -24,7 +24,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.web.socket.CloseStatus; @@ -42,7 +41,9 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -136,15 +137,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TbServiceInfoProvider serviceInfoProvider; - - @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") - private int maxSubscriptionsPerTenant; - @Value("${server.ws.limits.max_subscriptions_per_customer:0}") - private int maxSubscriptionsPerCustomer; - @Value("${server.ws.limits.max_subscriptions_per_regular_user:0}") - private int maxSubscriptionsPerRegularUser; - @Value("${server.ws.limits.max_subscriptions_per_public_user:0}") - private int maxSubscriptionsPerPublicUser; + @Autowired + private TbTenantProfileCache tenantProfileCache; private ConcurrentMap> tenantSubscriptionsMap = new ConcurrentHashMap<>(); private ConcurrentMap> customerSubscriptionsMap = new ConcurrentHashMap<>(); @@ -304,44 +298,50 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } private void processSessionClose(TelemetryWebSocketSessionRef sessionRef) { - String sessionId = "[" + sessionRef.getSessionId() + "]"; - if (maxSubscriptionsPerTenant > 0) { - Set tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); - synchronized (tenantSubscriptions) { - tenantSubscriptions.removeIf(subId -> subId.startsWith(sessionId)); - } - } - if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (maxSubscriptionsPerCustomer > 0) { - Set customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); - synchronized (customerSessions) { - customerSessions.removeIf(subId -> subId.startsWith(sessionId)); + var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + if(tenantProfileConfiguration != null) { + String sessionId = "[" + sessionRef.getSessionId() + "]"; + + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant() > 0) { + Set tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); + synchronized (tenantSubscriptions) { + tenantSubscriptions.removeIf(subId -> subId.startsWith(sessionId)); } } - if (maxSubscriptionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { - Set regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); - synchronized (regularUserSessions) { - regularUserSessions.removeIf(subId -> subId.startsWith(sessionId)); + if (sessionRef.getSecurityCtx().isCustomerUser()) { + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer() > 0) { + Set customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); + synchronized (customerSessions) { + customerSessions.removeIf(subId -> subId.startsWith(sessionId)); + } } - } - if (maxSubscriptionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { - Set publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); - synchronized (publicUserSessions) { - publicUserSessions.removeIf(subId -> subId.startsWith(sessionId)); + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + Set regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); + synchronized (regularUserSessions) { + regularUserSessions.removeIf(subId -> subId.startsWith(sessionId)); + } + } + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + Set publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); + synchronized (publicUserSessions) { + publicUserSessions.removeIf(subId -> subId.startsWith(sessionId)); + } } } } } private boolean processSubscription(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd) { + var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + String subId = "[" + sessionRef.getSessionId() + "]:[" + cmd.getCmdId() + "]"; try { - if (maxSubscriptionsPerTenant > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant() > 0) { Set tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSubscriptions) { if (cmd.isUnsubscribe()) { tenantSubscriptions.remove(subId); - } else if (tenantSubscriptions.size() < maxSubscriptionsPerTenant) { + } else if (tenantSubscriptions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant()) { tenantSubscriptions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max tenant subscriptions limit reached" @@ -353,12 +353,12 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (maxSubscriptionsPerCustomer > 0) { + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer() > 0) { Set customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { if (cmd.isUnsubscribe()) { customerSessions.remove(subId); - } else if (customerSessions.size() < maxSubscriptionsPerCustomer) { + } else if (customerSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer()) { customerSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max customer subscriptions limit reached" @@ -368,10 +368,10 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } } } - if (maxSubscriptionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { - if (regularUserSessions.size() < maxSubscriptionsPerRegularUser) { + if (regularUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser()) { regularUserSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max regular user subscriptions limit reached" @@ -381,10 +381,10 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } } } - if (maxSubscriptionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { - if (publicUserSessions.size() < maxSubscriptionsPerPublicUser) { + if (publicUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser()) { publicUserSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max public user subscriptions limit reached" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index a0fefea6cc..5efc041806 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -107,4 +107,11 @@ public class TenantProfile extends SearchTextBased implements H } } + public DefaultTenantProfileConfiguration getDefaultTenantConfiguration() { + if(this.profileData != null && this.profileData.getConfiguration().getType().equals(TenantProfileType.DEFAULT)) { + return (DefaultTenantProfileConfiguration) this.profileData.getConfiguration(); + } + return null; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index b9bd72b0db..4fe4811350 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -42,6 +42,23 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String rateLimitsTenantConfiguration; + private String rateLimitsCustomerConfiguration; + + private int wsLimitMaxSessionsPerTenant; + private int wsLimitMaxSessionsPerCustomer; + private int wsLimitMaxSessionsPerRegularUser; + private int wsLimitMaxSessionsPerPublicUser; + private int wsLimitQueuePerWsSession; + private long wsLimitMaxSubscriptionsPerTenant; + private long wsLimitMaxSubscriptionsPerCustomer; + private long wsLimitMaxSubscriptionsPerRegularUser; + private long wsLimitMaxSubscriptionsPerPublicUser; + private String wsLimitUpdatesPerSession; + + private String cassandraTenantLimitsConfiguration; + private boolean printTenantNames; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java index ac61a73f0c..02662e058d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java @@ -22,14 +22,18 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor; import org.thingsboard.server.dao.util.AsyncTaskContext; import org.thingsboard.server.dao.util.NoSqlAnyDao; +import org.thingsboard.server.dao.util.TenantRateLimitException; import javax.annotation.PreDestroy; import java.util.HashMap; @@ -45,6 +49,9 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< @Autowired private EntityService entityService; + @Autowired + private TbTenantProfileCache tenantProfileCache; + private Map tenantNamesCache = new HashMap<>(); private boolean printTenantNames; @@ -56,12 +63,10 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< @Value("${cassandra.query.dispatcher_threads:2}") int dispatcherThreads, @Value("${cassandra.query.callback_threads:4}") int callbackThreads, @Value("${cassandra.query.poll_ms:50}") long pollMs, - @Value("${cassandra.query.tenant_rate_limits.enabled}") boolean tenantRateLimitsEnabled, - @Value("${cassandra.query.tenant_rate_limits.configuration}") String tenantRateLimitsConfiguration, @Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") boolean printTenantNames, @Value("${cassandra.query.print_queries_freq:0}") int printQueriesFreq, @Autowired StatsFactory statsFactory) { - super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, tenantRateLimitsEnabled, tenantRateLimitsConfiguration, printQueriesFreq, statsFactory); + super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, printQueriesFreq, statsFactory); this.printTenantNames = printTenantNames; } @@ -97,7 +102,8 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< DefaultCounter counter = entry.getValue(); int rateLimitedRequests = counter.get(); counter.clear(); - if (printTenantNames) { + var profile = tenantProfileCache.get(tenantId).getDefaultTenantConfiguration(); + if (profile != null && profile.isPrintTenantNames()) { String name = tenantNamesCache.computeIfAbsent(tenantId, tId -> { try { return entityService.fetchEntityNameAsync(TenantId.SYS_TENANT_ID, tenantId).get(); @@ -137,4 +143,24 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< ); } + @Override + protected boolean checkRateLimits(CassandraStatementTask task, SettableFuture future) { + var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultTenantConfiguration(); + if (StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraTenantLimitsConfiguration())) { + if (task.getTenantId() == null) { + log.info("Invalid task received: {}", task); + } else if (!task.getTenantId().isNullUid()) { + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent( + task.getTenantId(), id -> new TbRateLimits(tenantProfileConfiguration.getCassandraTenantLimitsConfiguration()) + ); + if (!rateLimits.tryConsume()) { + stats.incrementRateLimitedTenant(task.getTenantId()); + stats.getTotalRateLimited().increment(); + future.setException(new TenantRateLimitException()); + return true; + } + } + } + return false; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index 157755db19..f201f36a09 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -66,9 +66,7 @@ public abstract class AbstractBufferedRateExecutor perTenantLimits = new ConcurrentHashMap<>(); + protected final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private final AtomicInteger printQueriesIdx = new AtomicInteger(0); @@ -76,7 +74,7 @@ public abstract class AbstractBufferedRateExecutor future); + @Override public F submit(T task) { SettableFuture settableFuture = create(); F result = wrap(task, settableFuture); - boolean perTenantLimitReached = false; - if (perTenantLimitsEnabled) { - if (task.getTenantId() == null) { - log.info("Invalid task received: {}", task); - } else if (!task.getTenantId().isNullUid()) { - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(task.getTenantId(), id -> new TbRateLimits(perTenantLimitsConfiguration)); - if (!rateLimits.tryConsume()) { - stats.incrementRateLimitedTenant(task.getTenantId()); - stats.getTotalRateLimited().increment(); - settableFuture.setException(new TenantRateLimitException()); - perTenantLimitReached = true; - } - } - } + boolean perTenantLimitReached = checkRateLimits(task, settableFuture); + if (!perTenantLimitReached) { try { stats.getTotalAdded().increment(); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html index 9af8d294de..1fdc6aba83 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html @@ -16,17 +16,21 @@ -->
- - - -
tenant-profile.profile-configuration
- - - - - - - + + + + + + + + + + + + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 13e3976299..78ba0ba1da 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -15,221 +15,371 @@ limitations under the License. --> -
- - tenant-profile.maximum-devices - - - {{ 'tenant-profile.maximum-devices-required' | translate}} - - - {{ 'tenant-profile.maximum-devices-range' | translate}} - - - - tenant-profile.maximum-assets - - - {{ 'tenant-profile.maximum-assets-required' | translate}} - - - {{ 'tenant-profile.maximum-assets-range' | translate}} - - - - tenant-profile.maximum-customers - - - {{ 'tenant-profile.maximum-customers-required' | translate}} - - - {{ 'tenant-profile.maximum-customers-range' | translate}} - - - - tenant-profile.maximum-users - - - {{ 'tenant-profile.maximum-users-required' | translate}} - - - {{ 'tenant-profile.maximum-users-range' | translate}} - - - - tenant-profile.maximum-dashboards - - - {{ 'tenant-profile.maximum-dashboards-required' | translate}} - - - {{ 'tenant-profile.maximum-dashboards-range' | translate}} - - - - tenant-profile.maximum-rule-chains - - - {{ 'tenant-profile.maximum-rule-chains-required' | translate}} - - - {{ 'tenant-profile.maximum-rule-chains-range' | translate}} - - - - tenant-profile.max-transport-messages - - - {{ 'tenant-profile.max-transport-messages-range' | translate}} - - - {{ 'tenant-profile.max-transport-messages-required' | translate}} - - - - tenant-profile.max-transport-data-points - - - {{ 'tenant-profile.max-transport-data-points-required' | translate}} - - - {{ 'tenant-profile.max-transport-data-points-range' | translate}} - - - - tenant-profile.max-r-e-executions - - - {{ 'tenant-profile.max-r-e-executions-required' | translate}} - - - {{ 'tenant-profile.max-r-e-executions-range' | translate}} - - - - tenant-profile.max-j-s-executions - - - {{ 'tenant-profile.max-j-s-executions-required' | translate}} - - - {{ 'tenant-profile.max-j-s-executions-range' | translate}} - - - - tenant-profile.max-d-p-storage-days - - - {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} - - - {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} - - - - tenant-profile.default-storage-ttl-days - - - {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} - - - {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} - - - - tenant-profile.max-rule-node-executions-per-message - - - {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} - - - {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} - - - - tenant-profile.max-emails - - - {{ 'tenant-profile.max-emails-required' | translate}} - - - {{ 'tenant-profile.max-emails-range' | translate}} - - - - tenant-profile.max-sms - - - {{ 'tenant-profile.max-sms-required' | translate}} - - - {{ 'tenant-profile.max-sms-range' | translate}} - - - - tenant-profile.max-created-alarms - - - {{ 'tenant-profile.max-created-alarms-required' | translate}} - - - {{ 'tenant-profile.max-created-alarms-range' | translate}} - - - - tenant-profile.transport-tenant-msg-rate-limit - - - - tenant-profile.transport-tenant-telemetry-msg-rate-limit - - - - tenant-profile.transport-tenant-telemetry-data-points-rate-limit - - - - tenant-profile.transport-device-msg-rate-limit - - - - tenant-profile.transport-device-telemetry-msg-rate-limit - - - - tenant-profile.transport-device-telemetry-data-points-rate-limit - - +
+ + + + Thingsboard rate limits + + + + tenant-profile.maximum-devices + + + {{ 'tenant-profile.maximum-devices-required' | translate}} + + + {{ 'tenant-profile.maximum-devices-range' | translate}} + + + + tenant-profile.maximum-assets + + + {{ 'tenant-profile.maximum-assets-required' | translate}} + + + {{ 'tenant-profile.maximum-assets-range' | translate}} + + + + tenant-profile.maximum-customers + + + {{ 'tenant-profile.maximum-customers-required' | translate}} + + + {{ 'tenant-profile.maximum-customers-range' | translate}} + + + + tenant-profile.maximum-users + + + {{ 'tenant-profile.maximum-users-required' | translate}} + + + {{ 'tenant-profile.maximum-users-range' | translate}} + + + + tenant-profile.maximum-dashboards + + + {{ 'tenant-profile.maximum-dashboards-required' | translate}} + + + {{ 'tenant-profile.maximum-dashboards-range' | translate}} + + + + tenant-profile.maximum-rule-chains + + + {{ 'tenant-profile.maximum-rule-chains-required' | translate}} + + + {{ 'tenant-profile.maximum-rule-chains-range' | translate}} + + + + tenant-profile.max-transport-messages + + + {{ 'tenant-profile.max-transport-messages-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-required' | translate}} + + + + tenant-profile.max-transport-data-points + + + {{ 'tenant-profile.max-transport-data-points-required' | translate}} + + + {{ 'tenant-profile.max-transport-data-points-range' | translate}} + + + + tenant-profile.max-r-e-executions + + + {{ 'tenant-profile.max-r-e-executions-required' | translate}} + + + {{ 'tenant-profile.max-r-e-executions-range' | translate}} + + + + tenant-profile.max-j-s-executions + + + {{ 'tenant-profile.max-j-s-executions-required' | translate}} + + + {{ 'tenant-profile.max-j-s-executions-range' | translate}} + + + + tenant-profile.max-d-p-storage-days + + + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} + + + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} + + + + tenant-profile.default-storage-ttl-days + + + {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} + + + {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} + + + + tenant-profile.max-rule-node-executions-per-message + + + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} + + + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} + + + + tenant-profile.max-emails + + + {{ 'tenant-profile.max-emails-required' | translate}} + + + {{ 'tenant-profile.max-emails-range' | translate}} + + + + tenant-profile.max-sms + + + {{ 'tenant-profile.max-sms-required' | translate}} + + + {{ 'tenant-profile.max-sms-range' | translate}} + + + + tenant-profile.max-created-alarms + + + {{ 'tenant-profile.max-created-alarms-required' | translate}} + + + {{ 'tenant-profile.max-created-alarms-range' | translate}} + + + + + + + + Server rate limits + + + + tenant-profile.tenant-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.customer-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + + + + + Transport rate limits + + + + tenant-profile.transport-tenant-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-tenant-telemetry-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-tenant-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-telemetry-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + + + + + Web Socket rate limits + + + + tenant-profile.ws-limit-max-sessions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-queue-per-session + + + {{ 'tenant-profile.too-small-value-one' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-regular-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-updates-per-session + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + + + + + Cassandra rate limits + + + + tenant-profile.cassandra-tenant-limits-configuration + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + {{ 'tenant-profile.cassandra-print-tenant-names' | translate }} + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index ad00df7476..6509ef0b20 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -35,6 +35,7 @@ import { isDefinedAndNotNull } from '@core/utils'; export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { defaultTenantProfileConfigurationFormGroup: FormGroup; + rateLimitsPattern = '^(((\\d+):(\\d+)),)*((\\d+):(\\d+))$'; private requiredValue: boolean; get required(): boolean { @@ -59,12 +60,12 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxUsers: [null, [Validators.required, Validators.min(0)]], maxDashboards: [null, [Validators.required, Validators.min(0)]], maxRuleChains: [null, [Validators.required, Validators.min(0)]], - transportTenantMsgRateLimit: [null, []], - transportTenantTelemetryMsgRateLimit: [null, []], - transportTenantTelemetryDataPointsRateLimit: [null, []], - transportDeviceMsgRateLimit: [null, []], - transportDeviceTelemetryMsgRateLimit: [null, []], - transportDeviceTelemetryDataPointsRateLimit: [null, []], + transportTenantMsgRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + transportTenantTelemetryMsgRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + transportTenantTelemetryDataPointsRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + transportDeviceMsgRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + transportDeviceTelemetryMsgRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + transportDeviceTelemetryDataPointsRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], maxTransportMessages: [null, [Validators.required, Validators.min(0)]], maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], maxREExecutions: [null, [Validators.required, Validators.min(0)]], @@ -74,7 +75,23 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxEmails: [null, [Validators.required, Validators.min(0)]], maxSms: [null, [Validators.required, Validators.min(0)]], maxCreatedAlarms: [null, [Validators.required, Validators.min(0)]], - defaultStorageTtlDays: [null, [Validators.required, Validators.min(0)]] + defaultStorageTtlDays: [null, [Validators.required, Validators.min(0)]], + + rateLimitsTenantConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], + rateLimitsCustomerConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], + + wsLimitMaxSessionsPerTenant: [null, [Validators.min(0)]], + wsLimitMaxSessionsPerCustomer: [null, [Validators.min(0)]], + wsLimitMaxSessionsPerPublicUser: [null, [Validators.min(0)]], + wsLimitQueuePerWsSession: [null, [Validators.min(0)]], + wsLimitMaxSubscriptionsPerTenant: [null, [Validators.min(0)]], + wsLimitMaxSubscriptionsPerCustomer: [null, [Validators.min(0)]], + wsLimitMaxSubscriptionsPerRegularUser: [null, [Validators.min(0)]], + wsLimitMaxSubscriptionsPerPublicUser: [null, [Validators.min(0)]], + wsLimitUpdatesPerSession: [null, [Validators.pattern(this.rateLimitsPattern)]], + + cassandraTenantLimitsConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], + printTenantNames: [null, []] }); this.defaultTenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { this.updateModel(); diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index e59832fc85..91e874d530 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -46,6 +46,24 @@ export interface DefaultTenantProfileConfiguration { maxRuleNodeExecutionsPerMessage: number; maxEmails: number; maxSms: number; + maxCreatedAlarms: number; + + rateLimitsTenantConfiguration: string; + rateLimitsCustomerConfiguration: string; + + wsLimitMaxSessionsPerTenant: number; + wsLimitMaxSessionsPerCustomer: number; + wsLimitMaxSessionsPerRegularUser: number; + wsLimitMaxSessionsPerPublicUser: number; + wsLimitQueuePerWsSession: number; + wsLimitMaxSubscriptionsPerTenant: number; + wsLimitMaxSubscriptionsPerCustomer: number; + wsLimitMaxSubscriptionsPerRegularUser: number; + wsLimitMaxSubscriptionsPerPublicUser: number; + wsLimitUpdatesPerSession: string; + + cassandraTenantLimitsConfiguration: string; + printTenantNames: boolean; defaultStorageTtlDays: number; } @@ -76,7 +94,25 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxRuleNodeExecutionsPerMessage: 0, maxEmails: 0, maxSms: 0, - defaultStorageTtlDays: 0 + defaultStorageTtlDays: 0, + maxCreatedAlarms: 0, + + rateLimitsTenantConfiguration: '', + rateLimitsCustomerConfiguration: '', + + wsLimitMaxSessionsPerTenant: 0, + wsLimitMaxSessionsPerCustomer: 0, + wsLimitMaxSessionsPerRegularUser: 0, + wsLimitMaxSessionsPerPublicUser: 0, + wsLimitQueuePerWsSession: 500, + wsLimitMaxSubscriptionsPerTenant: 0, + wsLimitMaxSubscriptionsPerCustomer: 0, + wsLimitMaxSubscriptionsPerRegularUser: 0, + wsLimitMaxSubscriptionsPerPublicUser: 0, + wsLimitUpdatesPerSession: '' , + + cassandraTenantLimitsConfiguration: '', + printTenantNames: false }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3da7b4f4bb..837e47b3d6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2533,7 +2533,23 @@ "max-sms-range": "Maximum number of SMS sent can't be negative", "max-created-alarms": "Maximum number of alarms created (0 - unlimited)", "max-created-alarms-required": "Maximum number of alarms created is required.", - "max-created-alarms-range": "Maximum number of alarms created can't be negative" + "max-created-alarms-range": "Maximum number of alarms created can't be negative", + "tenant-rest-limits": "Server limits of request for Tenant", + "customer-rest-limits": "Server limits of request for Customer", + "incorrect-pattern-for-rate-limits": "Here should be conformity of a number of requests per seconds divided by colon, comma separated. For example: 100:1,2000:60", + "too-small-value-zero": "Too small value specified, it should be more than 0", + "too-small-value-one": "Too small value specified, it should be more than 1", + "cassandra-tenant-limits-configuration": "Tenant limits configuration", + "ws-limit-max-sessions-per-tenant": "Max sessions per Tenant", + "ws-limit-max-sessions-per-customer": "Max sessions per Customer", + "ws-limit-max-sessions-per-public-user": "Max sessions per Public User", + "ws-limit-queue-per-session": "Limit queue per session", + "ws-limit-max-subscriptions-per-tenant": "Max subscriptions per Tenant", + "ws-limit-max-subscriptions-per-customer": "Max subscriptions per Customer", + "ws-limit-max-subscriptions-per-regular-user": "Max subscriptions per regular User", + "ws-limit-max-subscriptions-per-public-user": "Max subscriptions per public User", + "ws-limit-updates-per-session": "Constraint of updates per session", + "cassandra-print-tenant-names": "Print Tenant names to log" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", From 7c701906f87151d2e59ccce9386a69648eb71be9 Mon Sep 17 00:00:00 2001 From: AndrewVolosytnykhThingsboard Date: Fri, 28 May 2021 18:13:13 +0300 Subject: [PATCH 002/262] Code cleaning. JsonIgnore added to getter of TenantProfile --- .../server/config/RateLimitProcessingFilter.java | 11 +---------- .../server/controller/plugin/TbWebSocketHandler.java | 8 ++++---- .../telemetry/DefaultTelemetryWebSocketService.java | 4 ++-- .../thingsboard/server/common/data/TenantProfile.java | 6 ++++-- .../dao/nosql/CassandraBufferedRateExecutor.java | 4 ++-- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 381e767cce..6d531925c6 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -42,15 +42,6 @@ import java.util.concurrent.ConcurrentMap; @Component public class RateLimitProcessingFilter extends GenericFilterBean { -// @Value("${server.rest.limits.tenant.enabled:false}") -// private boolean perTenantLimitsEnabled; -// @Value("${server.rest.limits.tenant.configuration:}") -// private String perTenantLimitsConfiguration; -// @Value("${server.rest.limits.customer.enabled:false}") -// private boolean perCustomerLimitsEnabled; -// @Value("${server.rest.limits.customer.configuration:}") -// private String perCustomerLimitsConfiguration; - @Autowired private ThingsboardErrorResponseHandler errorResponseHandler; @@ -65,7 +56,7 @@ public class RateLimitProcessingFilter extends GenericFilterBean { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profile= tenantProfileCache.get(user.getTenantId()).getDefaultTenantConfiguration(); + var profile= tenantProfileCache.get(user.getTenantId()).getDefaultTenantProfileConfiguration(); if(profile != null) { if (StringUtils.isNotEmpty(profile.getRateLimitsTenantConfiguration())) { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index ea1b02eb7d..079ab58aeb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -118,7 +118,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (!checkLimits(session, sessionRef)) { return; } - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); if(tenantProfileConfiguration.getWsLimitQueuePerWsSession() <= 0) { tenantProfileConfiguration.setWsLimitQueuePerWsSession(500); } @@ -283,7 +283,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr log.debug("[{}] Processing {}", externalId, msg); String internalId = externalSessionMap.get(externalId); - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); if (internalId != null) { SessionMetaData sessionMd = internalSessionMap.get(internalId); @@ -346,7 +346,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr private boolean checkLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) throws Exception { var tenantProfileConfiguration = - tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); if(tenantProfileConfiguration == null) { return true; @@ -414,7 +414,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } private void cleanupLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) { - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); String sessionId = session.getId(); perSessionUpdateLimits.remove(sessionRef.getSessionId()); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index edb36c11c8..c35b562c55 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -298,7 +298,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } private void processSessionClose(TelemetryWebSocketSessionRef sessionRef) { - var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); if(tenantProfileConfiguration != null) { String sessionId = "[" + sessionRef.getSessionId() + "]"; @@ -332,7 +332,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } private boolean processSubscription(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd) { - var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); String subId = "[" + sessionRef.getSessionId() + "]:[" + cmd.getCmdId() + "]"; try { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 5efc041806..d5383d61a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -107,8 +107,10 @@ public class TenantProfile extends SearchTextBased implements H } } - public DefaultTenantProfileConfiguration getDefaultTenantConfiguration() { - if(this.profileData != null && this.profileData.getConfiguration().getType().equals(TenantProfileType.DEFAULT)) { + @JsonIgnore + public DefaultTenantProfileConfiguration getDefaultTenantProfileConfiguration() { + if(getProfileData().getConfiguration() != null && + getProfileData().getConfiguration().getType().equals(TenantProfileType.DEFAULT)) { return (DefaultTenantProfileConfiguration) this.profileData.getConfiguration(); } return null; diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java index 02662e058d..88d2daf15e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java @@ -102,7 +102,7 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< DefaultCounter counter = entry.getValue(); int rateLimitedRequests = counter.get(); counter.clear(); - var profile = tenantProfileCache.get(tenantId).getDefaultTenantConfiguration(); + var profile = tenantProfileCache.get(tenantId).getDefaultTenantProfileConfiguration(); if (profile != null && profile.isPrintTenantNames()) { String name = tenantNamesCache.computeIfAbsent(tenantId, tId -> { try { @@ -145,7 +145,7 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor< @Override protected boolean checkRateLimits(CassandraStatementTask task, SettableFuture future) { - var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultTenantConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultTenantProfileConfiguration(); if (StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraTenantLimitsConfiguration())) { if (task.getTenantId() == null) { log.info("Invalid task received: {}", task); From 1af249fcb4bb99feea1030f42e22ed8a65a3f025 Mon Sep 17 00:00:00 2001 From: AndrewVolosytnykhThingsboard Date: Tue, 1 Jun 2021 12:32:31 +0300 Subject: [PATCH 003/262] Updater added, improved server rate limits filter, cleaned redundant lines --- .../config/RateLimitProcessingFilter.java | 26 ++-- .../install/ThingsboardInstallService.java | 2 + .../update/DefaultDataUpdateService.java | 7 +- .../install/update/RateLimitsUpdater.java | 115 ++++++++++++++++++ .../server/common/msg/tools/TbRateLimits.java | 4 + .../tenant-profile-data.component.html | 13 -- 6 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 6d531925c6..75d15b18f2 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -56,24 +56,30 @@ public class RateLimitProcessingFilter extends GenericFilterBean { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profile= tenantProfileCache.get(user.getTenantId()).getDefaultTenantProfileConfiguration(); + var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultTenantProfileConfiguration(); - if(profile != null) { - if (StringUtils.isNotEmpty(profile.getRateLimitsTenantConfiguration())) { + if(profileConfiguration != null) { + if (user.isTenantAdmin() && StringUtils.isNotEmpty(profileConfiguration.getRateLimitsTenantConfiguration())) { + + TbRateLimits rateLimits = perTenantLimits.get(user.getTenantId()); + if(rateLimits == null || !rateLimits.getCurrentConfig().equals(profileConfiguration.getRateLimitsTenantConfiguration())) { + rateLimits = new TbRateLimits(profileConfiguration.getRateLimitsTenantConfiguration()); + perTenantLimits.put(user.getTenantId(), rateLimits); + } - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent( - user.getTenantId(), id -> new TbRateLimits(profile.getRateLimitsTenantConfiguration()) - ); if (!rateLimits.tryConsume()) { errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response); return; } } - if (StringUtils.isNotEmpty(profile.getRateLimitsCustomerConfiguration())) { + if (user.isCustomerUser() && StringUtils.isNotEmpty(profileConfiguration.getRateLimitsCustomerConfiguration())) { + + TbRateLimits rateLimits = perCustomerLimits.get(user.getCustomerId()); + if(rateLimits == null || !rateLimits.getCurrentConfig().equals(profileConfiguration.getRateLimitsCustomerConfiguration())) { + rateLimits = new TbRateLimits(profileConfiguration.getRateLimitsCustomerConfiguration()); + perCustomerLimits.put(user.getCustomerId(), rateLimits); + } - TbRateLimits rateLimits = perCustomerLimits.computeIfAbsent( - user.getCustomerId(), id -> new TbRateLimits(profile.getRateLimitsCustomerConfiguration()) - ); if (!rateLimits.tryConsume()) { errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response); return; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 267fb31ad7..9f9e78dc3d 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -203,6 +203,8 @@ public class ThingsboardInstallService { log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; + case "3.3.0": + dataUpdateService.updateData("3.3.0"); default: throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b91c46417c..9d78fed87c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.profile.TbDeviceProfileNode; import org.thingsboard.rule.engine.profile.TbDeviceProfileNodeConfiguration; import org.thingsboard.server.common.data.EntityView; @@ -48,7 +49,6 @@ import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.service.install.InstallScripts; import java.util.ArrayList; @@ -88,6 +88,9 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private AlarmDao alarmDao; + @Autowired + private RateLimitsUpdater rateLimitsUpdater; + @Override public void updateData(String fromVersion) throws Exception { switch (fromVersion) { @@ -108,6 +111,8 @@ public class DefaultDataUpdateService implements DataUpdateService { tenantsDefaultEdgeRuleChainUpdater.updateEntities(null); tenantsAlarmsCustomerUpdater.updateEntities(null); break; + case "3.3.0": + rateLimitsUpdater.updateEntities(null); default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java new file mode 100644 index 0000000000..aa11a12ff1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -0,0 +1,115 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.update; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.tenant.TenantProfileService; + +@Component +class RateLimitsUpdater extends PaginatedUpdater { + @Value("${server.rest.limits.tenant.enabled}") + boolean tenantServerRateLimitsEnabled; + @Value("${server.rest.limits.customer.enabled}") + boolean customerServerRateLimitsEnabled; + @Value("${server.rest.limits.tenant.configuration}") + String tenantServerRateLimitsConfiguration; + @Value("${server.rest.limits.customer.configuration}") + String customerServerRateLimitsConfiguration; + + @Value("${server.ws.limits.max_sessions_per_tenant}") + private int wsLimitMaxSessionsPerTenant; + @Value("${server.ws.limits.max_sessions_per_customer}") + private int wsLimitMaxSessionsPerCustomer; + @Value("${server.ws.limits.max_sessions_per_regular_user}") + private int wsLimitMaxSessionsPerRegularUser; + @Value("${server.ws.limits.max_sessions_per_public_user}") + private int wsLimitMaxSessionsPerPublicUser; + @Value("${server.ws.limits.max_queue_per_ws_session}") + private int wsLimitQueuePerWsSession; + @Value("${server.ws.limits.max_subscriptions_per_tenant}") + private long wsLimitMaxSubscriptionsPerTenant; + @Value("${server.ws.limits.max_subscriptions_per_customer}") + private long wsLimitMaxSubscriptionsPerCustomer; + @Value("${server.ws.limits.max_subscriptions_per_regular_user}") + private long wsLimitMaxSubscriptionsPerRegularUser; + @Value("${server.ws.limits.max_subscriptions_per_public_user}") + private long wsLimitMaxSubscriptionsPerPublicUser; + @Value("${server.ws.limits.max_updates_per_session}") + private String wsLimitUpdatesPerSession; + + @Value("${cassandra.query.tenant_rate_limits.enabled}") + private boolean cassandraLimitsIsEnabled; + @Value("${cassandra.query.tenant_rate_limits.configuration}") + private String cassandraTenantLimitsConfiguration; + @Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") + private boolean printTenantNames; + + @Autowired + private TenantProfileService tenantProfileService; + + @Override + protected boolean forceReportTotal() { + return true; + } + + @Override + protected String getName() { + return "Rate limits updater"; + } + + @Override + protected PageData findEntities(String id, PageLink pageLink) { + return tenantProfileService.findTenantProfiles(null, pageLink); + } + + @Override + protected void updateEntity(TenantProfile entity) { + var profileConfiguration = entity.getDefaultTenantProfileConfiguration(); + if (profileConfiguration != null) { + if (tenantServerRateLimitsEnabled && StringUtils.isNotEmpty(tenantServerRateLimitsConfiguration)) { + profileConfiguration.setRateLimitsTenantConfiguration(tenantServerRateLimitsConfiguration); + } + if (customerServerRateLimitsEnabled && StringUtils.isNotEmpty(customerServerRateLimitsConfiguration)) { + profileConfiguration.setRateLimitsCustomerConfiguration(customerServerRateLimitsConfiguration); + } + + profileConfiguration.setWsLimitMaxSessionsPerTenant(wsLimitMaxSessionsPerTenant); + profileConfiguration.setWsLimitMaxSessionsPerCustomer(wsLimitMaxSessionsPerCustomer); + profileConfiguration.setWsLimitMaxSessionsPerPublicUser(wsLimitMaxSessionsPerPublicUser); + profileConfiguration.setWsLimitMaxSessionsPerRegularUser(wsLimitMaxSessionsPerRegularUser); + profileConfiguration.setWsLimitMaxSubscriptionsPerTenant(wsLimitMaxSubscriptionsPerTenant); + profileConfiguration.setWsLimitMaxSubscriptionsPerCustomer(wsLimitMaxSubscriptionsPerCustomer); + profileConfiguration.setWsLimitMaxSubscriptionsPerPublicUser(wsLimitMaxSubscriptionsPerPublicUser); + profileConfiguration.setWsLimitMaxSubscriptionsPerRegularUser(wsLimitMaxSubscriptionsPerRegularUser); + profileConfiguration.setWsLimitQueuePerWsSession(wsLimitQueuePerWsSession); + + if (StringUtils.isNotEmpty(wsLimitUpdatesPerSession)) { + profileConfiguration.setWsLimitUpdatesPerSession(wsLimitUpdatesPerSession); + } + + if (cassandraLimitsIsEnabled) { + profileConfiguration.setCassandraTenantLimitsConfiguration(cassandraTenantLimitsConfiguration); + profileConfiguration.setPrintTenantNames(printTenantNames); + } + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index dc69406fe1..80fbaea65f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -19,6 +19,7 @@ import io.github.bucket4j.Bandwidth; import io.github.bucket4j.Bucket4j; import io.github.bucket4j.local.LocalBucket; import io.github.bucket4j.local.LocalBucketBuilder; +import lombok.Getter; import java.time.Duration; @@ -27,9 +28,12 @@ import java.time.Duration; */ public class TbRateLimits { private final LocalBucket bucket; + @Getter + private final String currentConfig; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); + currentConfig = limitsConfiguration; boolean initialized = false; for (String limitSrc : limitsConfiguration.split(",")) { long capacity = Long.parseLong(limitSrc.split(":")[0]); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html index 1fdc6aba83..4a979fb0cb 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html @@ -16,19 +16,6 @@ -->
- - - - - - - - - - - - - From d166353ce280802e69895dd78b155b7cd5cfbc3e Mon Sep 17 00:00:00 2001 From: AndrewVolosytnykhThingsboard Date: Wed, 2 Jun 2021 10:55:07 +0300 Subject: [PATCH 004/262] Refactoring of versions to update, correct update of tenant profile --- .../thingsboard/server/install/ThingsboardInstallService.java | 2 -- .../service/install/update/DefaultDataUpdateService.java | 3 +-- .../server/service/install/update/RateLimitsUpdater.java | 2 ++ 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 9f9e78dc3d..267fb31ad7 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -203,8 +203,6 @@ public class ThingsboardInstallService { log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; - case "3.3.0": - dataUpdateService.updateData("3.3.0"); default: throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 9d78fed87c..51a5fec601 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -110,9 +110,8 @@ public class DefaultDataUpdateService implements DataUpdateService { log.info("Updating data from version 3.2.2 to 3.3.0 ..."); tenantsDefaultEdgeRuleChainUpdater.updateEntities(null); tenantsAlarmsCustomerUpdater.updateEntities(null); - break; - case "3.3.0": rateLimitsUpdater.updateEntities(null); + break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index aa11a12ff1..e333641da3 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -110,6 +110,8 @@ class RateLimitsUpdater extends PaginatedUpdater { profileConfiguration.setCassandraTenantLimitsConfiguration(cassandraTenantLimitsConfiguration); profileConfiguration.setPrintTenantNames(printTenantNames); } + + tenantProfileService.saveTenantProfile(null, entity); } } } From e080005d6e97b4d908022d779cf5845f1d324f24 Mon Sep 17 00:00:00 2001 From: AndrewVolosytnykhThingsboard Date: Thu, 3 Jun 2021 15:06:56 +0300 Subject: [PATCH 005/262] Removed redundant setting of correct value for queue size per session --- .../server/controller/plugin/TbWebSocketHandler.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 079ab58aeb..ecf9a70b45 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -119,9 +119,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr return; } var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - if(tenantProfileConfiguration.getWsLimitQueuePerWsSession() <= 0) { - tenantProfileConfiguration.setWsLimitQueuePerWsSession(500); - } + internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsLimitQueuePerWsSession())); externalSessionMap.put(externalSessionId, internalSessionId); From ec10a425b878d155417705ca5e536ae4f951070c Mon Sep 17 00:00:00 2001 From: Rene Moser Date: Tue, 30 Nov 2021 17:01:40 +0100 Subject: [PATCH 006/262] Add option to configure spring's server.forward_headers_strategy --- application/src/main/resources/thingsboard.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 69bfd06bc8..813e75ee57 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -19,6 +19,8 @@ server: address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port port: "${HTTP_BIND_PORT:8080}" + # Server forward headers strategy + forward_headers_strategy: "${HTTP_FORWARD_HEADERS_STRATEGY:NONE}" # Server SSL configuration ssl: # Enable/disable SSL support From 3351ced60aceb7a4c3e7e5b6010190b99201ae7a Mon Sep 17 00:00:00 2001 From: desoliture Date: Tue, 4 Jan 2022 17:46:46 +0200 Subject: [PATCH 007/262] add backend support for dynamic values for schedules in alarm rules --- .../device/profile/CustomTimeSchedule.java | 3 ++ .../device/profile/SpecificTimeSchedule.java | 3 ++ .../transport/adaptor/JsonConverter.java | 4 ++ .../rule/engine/profile/AlarmRuleState.java | 41 ++++++++++++++----- .../rule/engine/profile/ProfileState.java | 24 +++++++++++ 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java index 89bf54eb25..83e8b3874e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import org.thingsboard.server.common.data.query.DynamicValue; import java.util.List; @@ -25,6 +26,8 @@ public class CustomTimeSchedule implements AlarmSchedule { private String timezone; private List items; + private DynamicValue dynamicValue; + @Override public AlarmScheduleType getType() { return AlarmScheduleType.CUSTOM; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java index 9ef17f25b0..57e57a1b24 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import org.thingsboard.server.common.data.query.DynamicValue; import java.util.List; import java.util.Set; @@ -28,6 +29,8 @@ public class SpecificTimeSchedule implements AlarmSchedule { private long startsOn; private long endsOn; + private DynamicValue dynamicValue; + @Override public AlarmScheduleType getType() { return AlarmScheduleType.SPECIFIC_TIME; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index be4143e388..af2ac148ce 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -573,6 +573,10 @@ public class JsonConverter { return JSON_PARSER.parse(json); } + public static T parse(String json, Class clazz) { + return fromJson(parse(json), clazz); + } + public static String toJson(JsonElement element) { return GSON.toJson(element); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 8ae6390b31..724b10c269 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.query.KeyFilterPredicate; import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.common.transport.adaptor.JsonConverter; import java.time.Instant; import java.time.ZoneId; @@ -115,7 +116,7 @@ class AlarmRuleState { } public AlarmEvalResult eval(DataSnapshot data) { - boolean active = isActive(data.getTs()); + boolean active = isActive(data, data.getTs()); switch (spec.getType()) { case SIMPLE: return (active && eval(alarmRule.getCondition(), data)) ? AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; @@ -128,7 +129,7 @@ class AlarmRuleState { } } - private boolean isActive(long eventTs) { + private boolean isActive(DataSnapshot data, long eventTs) { if (eventTs == 0L) { eventTs = System.currentTimeMillis(); } @@ -138,10 +139,28 @@ class AlarmRuleState { switch (alarmRule.getSchedule().getType()) { case ANY_TIME: return true; - case SPECIFIC_TIME: - return isActiveSpecific((SpecificTimeSchedule) alarmRule.getSchedule(), eventTs); - case CUSTOM: - return isActiveCustom((CustomTimeSchedule) alarmRule.getSchedule(), eventTs); + case SPECIFIC_TIME: { + SpecificTimeSchedule originalSchedule = (SpecificTimeSchedule) alarmRule.getSchedule(); + EntityKeyValue dynamicValue = getDynamicValue(data, originalSchedule.getDynamicValue()); + + if (dynamicValue != null) { + SpecificTimeSchedule schedule = JsonConverter.parse(dynamicValue.getJsonValue(), SpecificTimeSchedule.class); + originalSchedule = schedule == null ? originalSchedule : schedule; + } + + return isActiveSpecific(originalSchedule, eventTs); + } + case CUSTOM: { + CustomTimeSchedule originalSchedule = (CustomTimeSchedule) alarmRule.getSchedule(); + EntityKeyValue dynamicValue = getDynamicValue(data, originalSchedule.getDynamicValue()); + + if (dynamicValue != null) { + CustomTimeSchedule schedule = JsonConverter.parse(dynamicValue.getJsonValue(), CustomTimeSchedule.class); + originalSchedule = schedule == null ? originalSchedule : schedule; + } + + return isActiveCustom(originalSchedule, eventTs); + } default: throw new RuntimeException("Unsupported schedule type: " + alarmRule.getSchedule().getType()); } @@ -236,7 +255,7 @@ class AlarmRuleState { if (repeating.getPredicate().getDynamicValue() != null && repeating.getPredicate().getDynamicValue().getSourceAttribute() != null) { - EntityKeyValue repeatingKeyValue = getDynamicPredicateValue(data, repeating.getPredicate().getDynamicValue()); + EntityKeyValue repeatingKeyValue = getDynamicValue(data, repeating.getPredicate().getDynamicValue()); if (repeatingKeyValue != null) { repeatingTimes = repeatingKeyValue.getLngValue(); } @@ -257,7 +276,7 @@ class AlarmRuleState { if (duration.getPredicate().getDynamicValue() != null && duration.getPredicate().getDynamicValue().getSourceAttribute() != null) { - EntityKeyValue durationKeyValue = getDynamicPredicateValue(data, duration.getPredicate().getDynamicValue()); + EntityKeyValue durationKeyValue = getDynamicValue(data, duration.getPredicate().getDynamicValue()); if (durationKeyValue != null) { durationTimeInMs = timeUnit.toMillis(durationKeyValue.getLngValue()); } @@ -276,7 +295,7 @@ class AlarmRuleState { long requiredDurationInMs = resolveRequiredDurationInMs(dataSnapshot); if (requiredDurationInMs > 0 && state.getLastEventTs() > 0 && ts > state.getLastEventTs()) { long duration = state.getDuration() + (ts - state.getLastEventTs()); - if (isActive(ts)) { + if (isActive(dataSnapshot, ts)) { return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; } else { return AlarmEvalResult.FALSE; @@ -443,7 +462,7 @@ class AlarmRuleState { } private T getPredicateValue(DataSnapshot data, FilterPredicateValue value, AlarmConditionFilter filter, Function transformFunction) { - EntityKeyValue ekv = getDynamicPredicateValue(data, value.getDynamicValue()); + EntityKeyValue ekv = getDynamicValue(data, value.getDynamicValue()); if (ekv != null) { T result = transformFunction.apply(ekv); if (result != null) { @@ -457,7 +476,7 @@ class AlarmRuleState { } } - private EntityKeyValue getDynamicPredicateValue(DataSnapshot data, DynamicValue value) { + private EntityKeyValue getDynamicValue(DataSnapshot data, DynamicValue value) { EntityKeyValue ekv = null; if (value != null) { switch (value.getSourceType()) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java index 0cd4ed63ee..6d3c48892c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java @@ -27,6 +27,9 @@ import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DurationAlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.RepeatingAlarmConditionSpec; +import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; +import org.thingsboard.server.common.data.device.profile.SpecificTimeSchedule; +import org.thingsboard.server.common.data.device.profile.AlarmSchedule; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; @@ -77,6 +80,7 @@ class ProfileState { addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, ruleKeys); } addEntityKeysFromAlarmConditionSpec(alarmRule); + addScheduleDynamicValues(alarmRule.getSchedule()); })); if (alarm.getClearRule() != null) { var clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>()); @@ -91,6 +95,26 @@ class ProfileState { } } + private void addScheduleDynamicValues(AlarmSchedule schedule) { + DynamicValue dynamicValue = null; + switch (schedule.getType()) { + case SPECIFIC_TIME: + SpecificTimeSchedule specSchedule = (SpecificTimeSchedule) schedule; + dynamicValue = specSchedule.getDynamicValue(); + break; + case CUSTOM: + CustomTimeSchedule custSchedule = (CustomTimeSchedule) schedule; + dynamicValue = custSchedule.getDynamicValue(); + } + + if (dynamicValue != null) { + entityKeys.add( + new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, + dynamicValue.getSourceAttribute()) + ); + } + } + private void addEntityKeysFromAlarmConditionSpec(AlarmRule alarmRule) { AlarmConditionSpec spec = alarmRule.getCondition().getSpec(); if (spec == null) { From 2b203bfdd59cc7b2455411bc6230c2fdec9c5689 Mon Sep 17 00:00:00 2001 From: desoliture Date: Wed, 5 Jan 2022 13:24:53 +0200 Subject: [PATCH 008/262] fix notNull checking for schedule in ProfileState, rename variables and fix logic for parsing dynamicValues in AlarmRuleState --- .../rule/engine/profile/AlarmRuleState.java | 28 +++++++++++-------- .../rule/engine/profile/ProfileState.java | 14 ++++++---- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 724b10c269..c5ad4607b2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -140,26 +140,30 @@ class AlarmRuleState { case ANY_TIME: return true; case SPECIFIC_TIME: { - SpecificTimeSchedule originalSchedule = (SpecificTimeSchedule) alarmRule.getSchedule(); - EntityKeyValue dynamicValue = getDynamicValue(data, originalSchedule.getDynamicValue()); + SpecificTimeSchedule defaultSchedule = (SpecificTimeSchedule) alarmRule.getSchedule(); + EntityKeyValue dynamicValue = getDynamicValue(data, defaultSchedule.getDynamicValue()); - if (dynamicValue != null) { - SpecificTimeSchedule schedule = JsonConverter.parse(dynamicValue.getJsonValue(), SpecificTimeSchedule.class); - originalSchedule = schedule == null ? originalSchedule : schedule; + SpecificTimeSchedule schedule; + try { + schedule = JsonConverter.parse(dynamicValue.getJsonValue(), SpecificTimeSchedule.class); + } catch (Exception e) { + schedule = defaultSchedule; } - return isActiveSpecific(originalSchedule, eventTs); + return isActiveSpecific(schedule, eventTs); } case CUSTOM: { - CustomTimeSchedule originalSchedule = (CustomTimeSchedule) alarmRule.getSchedule(); - EntityKeyValue dynamicValue = getDynamicValue(data, originalSchedule.getDynamicValue()); + CustomTimeSchedule defaultSchedule = (CustomTimeSchedule) alarmRule.getSchedule(); + EntityKeyValue dynamicValue = getDynamicValue(data, defaultSchedule.getDynamicValue()); - if (dynamicValue != null) { - CustomTimeSchedule schedule = JsonConverter.parse(dynamicValue.getJsonValue(), CustomTimeSchedule.class); - originalSchedule = schedule == null ? originalSchedule : schedule; + CustomTimeSchedule schedule; + try { + schedule = JsonConverter.parse(dynamicValue.getJsonValue(), CustomTimeSchedule.class); + } catch (Exception e) { + schedule = defaultSchedule; } - return isActiveCustom(originalSchedule, eventTs); + return isActiveCustom(schedule, eventTs); } default: throw new RuntimeException("Unsupported schedule type: " + alarmRule.getSchedule().getType()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java index 6d3c48892c..c8751d65f7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java @@ -80,7 +80,10 @@ class ProfileState { addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, ruleKeys); } addEntityKeysFromAlarmConditionSpec(alarmRule); - addScheduleDynamicValues(alarmRule.getSchedule()); + AlarmSchedule schedule = alarmRule.getSchedule(); + if (schedule != null) { + addScheduleDynamicValues(schedule); + } })); if (alarm.getClearRule() != null) { var clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>()); @@ -99,12 +102,13 @@ class ProfileState { DynamicValue dynamicValue = null; switch (schedule.getType()) { case SPECIFIC_TIME: - SpecificTimeSchedule specSchedule = (SpecificTimeSchedule) schedule; - dynamicValue = specSchedule.getDynamicValue(); + SpecificTimeSchedule specificTimeSchedule = (SpecificTimeSchedule) schedule; + dynamicValue = specificTimeSchedule.getDynamicValue(); break; case CUSTOM: - CustomTimeSchedule custSchedule = (CustomTimeSchedule) schedule; - dynamicValue = custSchedule.getDynamicValue(); + CustomTimeSchedule customTimeSchedule = (CustomTimeSchedule) schedule; + dynamicValue = customTimeSchedule.getDynamicValue(); + break; } if (dynamicValue != null) { From 46e301b2dbc5bc9aaa6e8850961ee5fad1f08093 Mon Sep 17 00:00:00 2001 From: desoliture Date: Wed, 5 Jan 2022 18:14:37 +0200 Subject: [PATCH 009/262] Refactoring for alarm rule schedule add getDynamicValue method to AlarmSchedule interface, refactor main isActive processing methods for work with schedules and dynamic values --- .../data/device/profile/AlarmSchedule.java | 3 ++ .../data/device/profile/AnyTimeSchedule.java | 7 +++ .../device/profile/SpecificTimeSchedule.java | 1 - .../rule/engine/profile/AlarmRuleState.java | 47 ++++++++----------- .../rule/engine/profile/ProfileState.java | 15 +----- 5 files changed, 31 insertions(+), 42 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java index 1d5eceb1aa..1fc39ef147 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.query.DynamicValue; import java.io.Serializable; @@ -34,4 +35,6 @@ public interface AlarmSchedule extends Serializable { AlarmScheduleType getType(); + DynamicValue getDynamicValue(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java index e02086902f..94eea60950 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.device.profile; +import org.thingsboard.server.common.data.query.DynamicValue; + public class AnyTimeSchedule implements AlarmSchedule { @Override @@ -22,4 +24,9 @@ public class AnyTimeSchedule implements AlarmSchedule { return AlarmScheduleType.ANY_TIME; } + @Override + public DynamicValue getDynamicValue() { + return null; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java index 57e57a1b24..0b8dfde791 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java @@ -18,7 +18,6 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.query.DynamicValue; -import java.util.List; import java.util.Set; @Data diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index c5ad4607b2..abcc414206 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.device.profile.AlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.AlarmConditionSpecType; import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; +import org.thingsboard.server.common.data.device.profile.AlarmSchedule; import org.thingsboard.server.common.data.device.profile.CustomTimeScheduleItem; import org.thingsboard.server.common.data.device.profile.DurationAlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.RepeatingAlarmConditionSpec; @@ -139,35 +140,27 @@ class AlarmRuleState { switch (alarmRule.getSchedule().getType()) { case ANY_TIME: return true; - case SPECIFIC_TIME: { - SpecificTimeSchedule defaultSchedule = (SpecificTimeSchedule) alarmRule.getSchedule(); - EntityKeyValue dynamicValue = getDynamicValue(data, defaultSchedule.getDynamicValue()); - - SpecificTimeSchedule schedule; - try { - schedule = JsonConverter.parse(dynamicValue.getJsonValue(), SpecificTimeSchedule.class); - } catch (Exception e) { - schedule = defaultSchedule; - } - - return isActiveSpecific(schedule, eventTs); - } - case CUSTOM: { - CustomTimeSchedule defaultSchedule = (CustomTimeSchedule) alarmRule.getSchedule(); - EntityKeyValue dynamicValue = getDynamicValue(data, defaultSchedule.getDynamicValue()); + case SPECIFIC_TIME: + return isActiveSpecific((SpecificTimeSchedule) getSchedule(data, alarmRule), eventTs); + case CUSTOM: + return isActiveCustom((CustomTimeSchedule) getSchedule(data, alarmRule), eventTs); + default: + throw new RuntimeException("Unsupported schedule type: " + alarmRule.getSchedule().getType()); + } + } - CustomTimeSchedule schedule; - try { - schedule = JsonConverter.parse(dynamicValue.getJsonValue(), CustomTimeSchedule.class); - } catch (Exception e) { - schedule = defaultSchedule; - } + private AlarmSchedule getSchedule(DataSnapshot data, AlarmRule alarmRule) { + AlarmSchedule schedule = alarmRule.getSchedule(); + EntityKeyValue dynamicValue = getDynamicValue(data, schedule.getDynamicValue()); - return isActiveCustom(schedule, eventTs); + if (dynamicValue != null) { + try { + return JsonConverter.parse(dynamicValue.getJsonValue(), alarmRule.getSchedule().getClass()); + } catch (Exception e) { + log.trace("Failed to parse AlarmSchedule from dynamicValue: {}", dynamicValue.getJsonValue(), e); } - default: - throw new RuntimeException("Unsupported schedule type: " + alarmRule.getSchedule().getType()); } + return schedule; } private boolean isActiveSpecific(SpecificTimeSchedule schedule, long eventTs) { @@ -252,7 +245,7 @@ class AlarmRuleState { long repeatingTimes = 0; AlarmConditionSpec alarmConditionSpec = getSpec(); AlarmConditionSpecType specType = alarmConditionSpec.getType(); - if(specType.equals(AlarmConditionSpecType.REPEATING)) { + if (specType.equals(AlarmConditionSpecType.REPEATING)) { RepeatingAlarmConditionSpec repeating = (RepeatingAlarmConditionSpec) spec; repeatingTimes = repeating.getPredicate().getDefaultValue(); @@ -272,7 +265,7 @@ class AlarmRuleState { long durationTimeInMs = 0; AlarmConditionSpec alarmConditionSpec = getSpec(); AlarmConditionSpecType specType = alarmConditionSpec.getType(); - if(specType.equals(AlarmConditionSpecType.DURATION)) { + if (specType.equals(AlarmConditionSpecType.DURATION)) { DurationAlarmConditionSpec duration = (DurationAlarmConditionSpec) spec; TimeUnit timeUnit = duration.getUnit(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java index c8751d65f7..dcfcea98ba 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java @@ -27,8 +27,6 @@ import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DurationAlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.RepeatingAlarmConditionSpec; -import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; -import org.thingsboard.server.common.data.device.profile.SpecificTimeSchedule; import org.thingsboard.server.common.data.device.profile.AlarmSchedule; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; @@ -99,18 +97,7 @@ class ProfileState { } private void addScheduleDynamicValues(AlarmSchedule schedule) { - DynamicValue dynamicValue = null; - switch (schedule.getType()) { - case SPECIFIC_TIME: - SpecificTimeSchedule specificTimeSchedule = (SpecificTimeSchedule) schedule; - dynamicValue = specificTimeSchedule.getDynamicValue(); - break; - case CUSTOM: - CustomTimeSchedule customTimeSchedule = (CustomTimeSchedule) schedule; - dynamicValue = customTimeSchedule.getDynamicValue(); - break; - } - + DynamicValue dynamicValue = schedule.getDynamicValue(); if (dynamicValue != null) { entityKeys.add( new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, From 03256deee78c9b30d2b13a646c5c53db9a03bfbc Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 6 Jan 2022 14:05:12 +0200 Subject: [PATCH 010/262] add corresponding tests for schedule from dynamic values --- .../profile/TbDeviceProfileNodeTest.java | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index b2c79102c4..63dc9a81d2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -44,6 +44,8 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.DurationAlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.RepeatingAlarmConditionSpec; +import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; +import org.thingsboard.server.common.data.device.profile.CustomTimeScheduleItem; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -71,6 +73,7 @@ import java.math.RoundingMode; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.ArrayList; import java.util.Optional; import java.util.TreeMap; import java.util.UUID; @@ -1086,6 +1089,182 @@ public class TbDeviceProfileNodeTest { verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); } + @Test + public void testActiveAlarmScheduleFromDynamicValuesWhenDefaultScheduleIsInactive() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(deviceProfileId); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + Device device = new Device(); + device.setId(deviceId); + device.setCustomerId(customerId); + + AttributeKvCompositeKey compositeKeyActiveSchedule = new AttributeKvCompositeKey( + EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "dynamicValueActiveSchedule" + ); + + AttributeKvEntity attributeKvEntityActiveSchedule = new AttributeKvEntity(); + attributeKvEntityActiveSchedule.setId(compositeKeyActiveSchedule); + attributeKvEntityActiveSchedule.setJsonValue( + "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":8.64e+7,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" + ); + attributeKvEntityActiveSchedule.setLastUpdateTs(0L); + + AttributeKvEntry entryActiveSchedule = attributeKvEntityActiveSchedule.toData(); + + ListenableFuture> listListenableFutureActiveSchedule = + Futures.immediateFuture(Collections.singletonList(entryActiveSchedule)); + + AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); + highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); + highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setValue(new FilterPredicateValue<>( + 0.0, + null, + null + )); + highTempFilter.setPredicate(highTemperaturePredicate); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Collections.singletonList(highTempFilter)); + + CustomTimeSchedule schedule = new CustomTimeSchedule(); + schedule.setItems(Collections.emptyList()); + schedule.setDynamicValue(new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "dynamicValueActiveSchedule", false)); + + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + alarmRule.setSchedule(schedule); + DeviceProfileAlarm deviceProfileAlarmActiveSchedule = new DeviceProfileAlarm(); + deviceProfileAlarmActiveSchedule.setId("highTemperatureAlarmID"); + deviceProfileAlarmActiveSchedule.setAlarmType("highTemperatureAlarm"); + deviceProfileAlarmActiveSchedule.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(deviceProfileAlarmActiveSchedule)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "highTemperatureAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(listListenableFutureActiveSchedule); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 35); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).enqueueForTellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + + @Test + public void testInactiveAlarmScheduleFromDynamicValuesWhenDefaultScheduleIsActive() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(deviceProfileId); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + Device device = new Device(); + device.setId(deviceId); + device.setCustomerId(customerId); + + AttributeKvCompositeKey compositeKeyInactiveSchedule = new AttributeKvCompositeKey( + EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "dynamicValueInactiveSchedule" + ); + + AttributeKvEntity attributeKvEntityInactiveSchedule = new AttributeKvEntity(); + attributeKvEntityInactiveSchedule.setId(compositeKeyInactiveSchedule); + attributeKvEntityInactiveSchedule.setJsonValue( + "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":false,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":0}],\"dynamicValue\":null}" + ); + + attributeKvEntityInactiveSchedule.setLastUpdateTs(0L); + + AttributeKvEntry entryInactiveSchedule = attributeKvEntityInactiveSchedule.toData(); + + ListenableFuture> listListenableFutureInactiveSchedule = + Futures.immediateFuture(Collections.singletonList(entryInactiveSchedule)); + + AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); + highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); + highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setValue(new FilterPredicateValue<>( + 0.0, + null, + null + )); + + highTempFilter.setPredicate(highTemperaturePredicate); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Collections.singletonList(highTempFilter)); + + CustomTimeSchedule schedule = new CustomTimeSchedule(); + + List items = new ArrayList<>(); + for (int i = 0; i < 7; i++) { + CustomTimeScheduleItem item = new CustomTimeScheduleItem(); + item.setEnabled(true); + item.setDayOfWeek(i + 1); + item.setEndsOn(0); + item.setStartsOn(0); + items.add(item); + } + + schedule.setItems(items); + schedule.setDynamicValue(new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "dynamicValueInactiveSchedule", false)); + + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + alarmRule.setSchedule(schedule); + DeviceProfileAlarm deviceProfileAlarmNonactiveSchedule = new DeviceProfileAlarm(); + deviceProfileAlarmNonactiveSchedule.setId("highTemperatureAlarmID"); + deviceProfileAlarmNonactiveSchedule.setAlarmType("highTemperatureAlarm"); + deviceProfileAlarmNonactiveSchedule.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(deviceProfileAlarmNonactiveSchedule)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "highTemperatureAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(listListenableFutureInactiveSchedule); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 35); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx, Mockito.never()).enqueueForTellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } @Test public void testCurrentCustomersAttributeForDynamicValue() throws Exception { From 8f485010f0b0a4328c29cb23f57c37c1a705e3ee Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 6 Jan 2022 16:14:46 +0200 Subject: [PATCH 011/262] refactoring --- .../rule/engine/profile/TbDeviceProfileNodeTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 63dc9a81d2..30e2943110 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -1246,14 +1246,11 @@ public class TbDeviceProfileNodeTest { .thenReturn(Futures.immediateFuture(Collections.emptyList())); Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "highTemperatureAlarm")) .thenReturn(Futures.immediateFuture(null)); - Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg()); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) .thenReturn(listListenableFutureInactiveSchedule); TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); - Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) - .thenReturn(theMsg); ObjectNode data = mapper.createObjectNode(); data.put("temperature", 35); From 6875df9dab6750d2ba0fe2b55c844e639fbfc343 Mon Sep 17 00:00:00 2001 From: Kalutka Zhenya Date: Mon, 10 Jan 2022 16:49:15 +0200 Subject: [PATCH 012/262] Added Dynamic value to alarm schedule --- .../alarm/alarm-schedule.component.html | 40 ++++++++++++++++++- .../profile/alarm/alarm-schedule.component.ts | 28 +++++++++++-- ui-ngx/src/app/shared/models/device.models.ts | 4 ++ .../assets/locale/locale.constant-en_US.json | 1 + 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html index bf09b33d76..5d63b48f15 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html @@ -73,8 +73,44 @@
-
device-profile.schedule-days
- + + +
+ +
+ {{'filter.dynamic-value' | translate}} +
+ + +
+ + +
+
+
+ + + + + {{'filter.no-dynamic-value' | translate}} + + + {{dynamicValueSourceTypeTranslations.get(sourceType) | translate}} + + + +
+
+ + + + +
+
+
+ + +
device-profile.schedule-days
diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts index d5c9f02876..6fd679f3f7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts @@ -40,6 +40,12 @@ import { import { isDefined, isDefinedAndNotNull } from '@core/utils'; import { MatCheckboxChange } from '@angular/material/checkbox'; import { getDefaultTimezone } from '@shared/models/time/time.models'; +import { + DynamicValueSourceType, + dynamicValueSourceTypeTranslationMap, + getDynamicSourcesForAllowUser +} from '@shared/models/query/query.models'; +import { emit } from 'cluster'; @Component({ selector: 'tb-alarm-schedule', @@ -64,7 +70,8 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, alarmScheduleTypes = Object.keys(AlarmScheduleType); alarmScheduleType = AlarmScheduleType; alarmScheduleTypeTranslate = AlarmScheduleTypeTranslationMap; - + dynamicValueSourceTypes: DynamicValueSourceType[] = getDynamicSourcesForAllowUser(false); + dynamicValueSourceTypeTranslations = dynamicValueSourceTypeTranslationMap; dayOfWeekTranslationsArray = dayOfWeekTranslations; allDays = Array(7).fill(0).map((x, i) => i); @@ -91,8 +98,19 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, daysOfWeek: this.fb.array(new Array(7).fill(false), this.validateDayOfWeeks), startsOn: [0, Validators.required], endsOn: [0, Validators.required], - items: this.fb.array(Array.from({length: 7}, (value, i) => this.defaultItemsScheduler(i)), this.validateItems) + items: this.fb.array(Array.from({length: 7}, (value, i) => this.defaultItemsScheduler(i)), this.validateItems), + dynamicValue: this.fb.group({ + sourceType: [null], + sourceAttribute: [null] + }) }); + + this.alarmScheduleForm.get('dynamicValue.sourceType').valueChanges.subscribe((sourceType) => { + if (!sourceType) { + this.alarmScheduleForm.get('dynamicValue.sourceAttribute').patchValue('', {emitEvent:false}); + } + }) + this.alarmScheduleForm.get('type').valueChanges.subscribe((type) => { const defaultTimezone = getDefaultTimezone(); this.alarmScheduleForm.reset({type, items: this.defaultItems, timezone: defaultTimezone}, {emitEvent: false}); @@ -177,7 +195,11 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, this.alarmScheduleForm.patchValue({ type: this.modelValue.type, timezone: this.modelValue.timezone, - items: alarmDays + items: alarmDays, + dynamicValue: { + sourceAttribute: this.modelValue.dynamicValue.sourceAttribute, + sourceType: this.modelValue.dynamicValue.sourceType + } }, {emitEvent: false}); } break; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 7fc3d70f78..ffb944803c 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -477,6 +477,10 @@ export const AlarmScheduleTypeTranslationMap = new Map Date: Tue, 11 Jan 2022 14:38:07 +0200 Subject: [PATCH 013/262] Updated dynamic value logic --- .../home/components/home-components.module.ts | 4 +- .../alarm/alarm-dynamic-value.component.html | 37 ++++++++++ .../alarm/alarm-dynamic-value.component.ts | 68 +++++++++++++++++++ .../alarm/alarm-schedule.component.html | 41 +---------- .../profile/alarm/alarm-schedule.component.ts | 27 ++------ 5 files changed, 115 insertions(+), 62 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 73361eddce..6614edebe8 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -147,6 +147,7 @@ import { HOME_COMPONENTS_MODULE_TOKEN } from '@home/components/tokens'; import { DashboardStateComponent } from '@home/components/dashboard-page/dashboard-state.component'; +import { AlarmDynamicValue } from '@home/components/profile/alarm/alarm-dynamic-value.component'; @NgModule({ declarations: @@ -245,6 +246,7 @@ import { DashboardStateComponent } from '@home/components/dashboard-page/dashboa AlarmScheduleInfoComponent, DeviceProfileProvisionConfigurationComponent, AlarmScheduleComponent, + AlarmDynamicValue, AlarmDurationPredicateValueComponent, DeviceWizardDialogComponent, AlarmScheduleDialogComponent, @@ -356,11 +358,11 @@ import { DashboardStateComponent } from '@home/components/dashboard-page/dashboa DeviceWizardDialogComponent, AlarmScheduleInfoComponent, AlarmScheduleComponent, + AlarmDynamicValue, AlarmScheduleDialogComponent, AlarmDurationPredicateValueComponent, EditAlarmDetailsDialogComponent, DeviceProfileProvisionConfigurationComponent, - AlarmScheduleComponent, SmsProviderConfigurationComponent, AwsSnsProviderConfigurationComponent, TwilioSmsProviderConfigurationComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html new file mode 100644 index 0000000000..6b2852cd68 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html @@ -0,0 +1,37 @@ + + +
+ +
+ {{'filter.dynamic-value' | translate}} +
+ + +
+ + +
+
+
+ + + + + {{'filter.no-dynamic-value' | translate}} + + + {{dynamicValueSourceTypeTranslations.get(sourceType) | translate}} + + + +
+
+ + + + +
+
+
+ + diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts new file mode 100644 index 0000000000..cb4efc1480 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts @@ -0,0 +1,68 @@ +import { Component, forwardRef, OnInit } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALUE_ACCESSOR, +} from '@angular/forms'; +import { + DynamicValueSourceType, + dynamicValueSourceTypeTranslationMap, + getDynamicSourcesForAllowUser +} from '@shared/models/query/query.models'; + +@Component({ + selector: 'tb-alarm-dynamic-value', + templateUrl: './alarm-dynamic-value.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AlarmDynamicValue), + multi: true + }] +}) + +export class AlarmDynamicValue implements ControlValueAccessor, OnInit{ + public dynamicValue: FormGroup; + public dynamicValueSourceTypes: DynamicValueSourceType[] = getDynamicSourcesForAllowUser(false); + public dynamicValueSourceTypeTranslations = dynamicValueSourceTypeTranslationMap; + private propagateChange = (v: any) => { }; + + constructor(private fb: FormBuilder) { + } + + ngOnInit(): void { + this.dynamicValue = this.fb.group({ + sourceType: [null], + sourceAttribute: [null] + }) + + this.dynamicValue.get('sourceType').valueChanges.subscribe( + (sourceType) => { + if (!sourceType) { + this.dynamicValue.get('sourceAttribute').patchValue(null, {emitEvent: false}); + } + } + ); + + this.dynamicValue.valueChanges.subscribe(() => { + this.updateModel(); + }) + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + writeValue(dynamicValue: {sourceType: string, sourceAttribute: string}): void { + if(dynamicValue) { + this.dynamicValue.patchValue(dynamicValue, {emitEvent: false}); + } + } + + private updateModel() { + this.propagateChange(this.dynamicValue.value); + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html index 5d63b48f15..67a3ac69c0 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html @@ -34,6 +34,7 @@ formControlName="timezone">
+
device-profile.schedule-days
@@ -73,44 +74,8 @@
- - -
- -
- {{'filter.dynamic-value' | translate}} -
- - -
- - -
-
-
- - - - - {{'filter.no-dynamic-value' | translate}} - - - {{dynamicValueSourceTypeTranslations.get(sourceType) | translate}} - - - -
-
- - - - -
-
-
- - -
device-profile.schedule-days
+ +
device-profile.schedule-days
diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts index 6fd679f3f7..2deef61a6d 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.ts @@ -40,12 +40,6 @@ import { import { isDefined, isDefinedAndNotNull } from '@core/utils'; import { MatCheckboxChange } from '@angular/material/checkbox'; import { getDefaultTimezone } from '@shared/models/time/time.models'; -import { - DynamicValueSourceType, - dynamicValueSourceTypeTranslationMap, - getDynamicSourcesForAllowUser -} from '@shared/models/query/query.models'; -import { emit } from 'cluster'; @Component({ selector: 'tb-alarm-schedule', @@ -70,8 +64,6 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, alarmScheduleTypes = Object.keys(AlarmScheduleType); alarmScheduleType = AlarmScheduleType; alarmScheduleTypeTranslate = AlarmScheduleTypeTranslationMap; - dynamicValueSourceTypes: DynamicValueSourceType[] = getDynamicSourcesForAllowUser(false); - dynamicValueSourceTypeTranslations = dynamicValueSourceTypeTranslationMap; dayOfWeekTranslationsArray = dayOfWeekTranslations; allDays = Array(7).fill(0).map((x, i) => i); @@ -99,18 +91,9 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, startsOn: [0, Validators.required], endsOn: [0, Validators.required], items: this.fb.array(Array.from({length: 7}, (value, i) => this.defaultItemsScheduler(i)), this.validateItems), - dynamicValue: this.fb.group({ - sourceType: [null], - sourceAttribute: [null] - }) + dynamicValue: [null] }); - this.alarmScheduleForm.get('dynamicValue.sourceType').valueChanges.subscribe((sourceType) => { - if (!sourceType) { - this.alarmScheduleForm.get('dynamicValue.sourceAttribute').patchValue('', {emitEvent:false}); - } - }) - this.alarmScheduleForm.get('type').valueChanges.subscribe((type) => { const defaultTimezone = getDefaultTimezone(); this.alarmScheduleForm.reset({type, items: this.defaultItems, timezone: defaultTimezone}, {emitEvent: false}); @@ -176,7 +159,8 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, timezone: this.modelValue.timezone, daysOfWeek, startsOn: utcTimestampToTimeOfDay(this.modelValue.startsOn), - endsOn: utcTimestampToTimeOfDay(this.modelValue.endsOn) + endsOn: utcTimestampToTimeOfDay(this.modelValue.endsOn), + dynamicValue: this.modelValue.dynamicValue }, {emitEvent: false}); break; case AlarmScheduleType.CUSTOM: @@ -196,10 +180,7 @@ export class AlarmScheduleComponent implements ControlValueAccessor, Validator, type: this.modelValue.type, timezone: this.modelValue.timezone, items: alarmDays, - dynamicValue: { - sourceAttribute: this.modelValue.dynamicValue.sourceAttribute, - sourceType: this.modelValue.dynamicValue.sourceType - } + dynamicValue: this.modelValue.dynamicValue }, {emitEvent: false}); } break; From 877257202e9e96925d684e8d448259d0ceada56b Mon Sep 17 00:00:00 2001 From: desoliture Date: Tue, 11 Jan 2022 15:38:20 +0200 Subject: [PATCH 014/262] fix incorrect work if endsOn equals zero when choosing schedule to be all day active (from 00:00 to 00:00) startsOn and endsOn equals zero, in this case make endsOn equals 24hours in millis --- .../thingsboard/rule/engine/profile/AlarmRuleState.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index abcc414206..f997efdeb3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -182,7 +182,12 @@ class AlarmRuleState { for (CustomTimeScheduleItem item : schedule.getItems()) { if (item.getDayOfWeek() == dayOfWeek) { if (item.isEnabled()) { - return isActive(eventTs, zoneId, zdt, item.getStartsOn(), item.getEndsOn()); + long endsOn = item.getEndsOn(); + if (endsOn == 0) { + // 24 hours in milliseconds + endsOn = 86400000; + } + return isActive(eventTs, zoneId, zdt, item.getStartsOn(), endsOn); } else { return false; } From 7328012cb491b16eebb89f1c6c0380861bf94c7f Mon Sep 17 00:00:00 2001 From: Kalutka Zhenya Date: Tue, 11 Jan 2022 16:06:08 +0200 Subject: [PATCH 015/262] Added license headers --- .../alarm/alarm-dynamic-value.component.html | 17 +++++++++++++++++ .../alarm/alarm-dynamic-value.component.ts | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html index 6b2852cd68..50b2dee7a7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html @@ -1,3 +1,20 @@ +
diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts index cb4efc1480..ad4b8cc02c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts @@ -1,3 +1,19 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + import { Component, forwardRef, OnInit } from '@angular/core'; import { ControlValueAccessor, From 487d7165cc690a71f4a084a1b9c7979fcbe3e663 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 20 Jan 2022 14:26:23 +0200 Subject: [PATCH 016/262] Rate limits to tenant profile BE refactoring --- .../config/RateLimitProcessingFilter.java | 24 ++--- .../controller/plugin/TbWebSocketHandler.java | 40 ++++---- .../install/ThingsboardInstallService.java | 3 + .../update/DefaultDataUpdateService.java | 15 +-- .../install/update/PaginatedUpdater.java | 4 + .../install/update/RateLimitsUpdater.java | 94 +++++++++---------- .../DefaultTelemetryWebSocketService.java | 27 +++--- .../server/common/data/TenantProfile.java | 9 +- .../DefaultTenantProfileConfiguration.java | 27 +++--- .../server/common/msg/tools/TbRateLimits.java | 6 +- .../CassandraBufferedRateReadExecutor.java | 35 +------ .../CassandraBufferedRateWriteExecutor.java | 10 +- .../util/AbstractBufferedRateExecutor.java | 32 +++++-- 13 files changed, 151 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 75d15b18f2..7163ba3feb 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -48,22 +48,20 @@ public class RateLimitProcessingFilter extends GenericFilterBean { @Autowired private TbTenantProfileCache tenantProfileCache; - private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private ConcurrentMap perCustomerLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perCustomerLimits = new ConcurrentHashMap<>(); @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultTenantProfileConfiguration(); - - if(profileConfiguration != null) { - if (user.isTenantAdmin() && StringUtils.isNotEmpty(profileConfiguration.getRateLimitsTenantConfiguration())) { - + if (profileConfiguration != null) { + if (user.isTenantAdmin() && StringUtils.isNotEmpty(profileConfiguration.getTenantServerRestLimitsConfiguration())) { TbRateLimits rateLimits = perTenantLimits.get(user.getTenantId()); - if(rateLimits == null || !rateLimits.getCurrentConfig().equals(profileConfiguration.getRateLimitsTenantConfiguration())) { - rateLimits = new TbRateLimits(profileConfiguration.getRateLimitsTenantConfiguration()); + if (rateLimits == null || !rateLimits.getConfiguration().equals(profileConfiguration.getTenantServerRestLimitsConfiguration())) { + // fixme: or maybe handle component lifecycle event ? + rateLimits = new TbRateLimits(profileConfiguration.getTenantServerRestLimitsConfiguration()); perTenantLimits.put(user.getTenantId(), rateLimits); } @@ -71,12 +69,10 @@ public class RateLimitProcessingFilter extends GenericFilterBean { errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response); return; } - } - if (user.isCustomerUser() && StringUtils.isNotEmpty(profileConfiguration.getRateLimitsCustomerConfiguration())) { - + } else if (user.isCustomerUser() && StringUtils.isNotEmpty(profileConfiguration.getCustomerServerRestLimitsConfiguration())) { TbRateLimits rateLimits = perCustomerLimits.get(user.getCustomerId()); - if(rateLimits == null || !rateLimits.getCurrentConfig().equals(profileConfiguration.getRateLimitsCustomerConfiguration())) { - rateLimits = new TbRateLimits(profileConfiguration.getRateLimitsCustomerConfiguration()); + if (rateLimits == null || !rateLimits.getConfiguration().equals(profileConfiguration.getCustomerServerRestLimitsConfiguration())) { + rateLimits = new TbRateLimits(profileConfiguration.getCustomerServerRestLimitsConfiguration()); perCustomerLimits.put(user.getCustomerId(), rateLimits); } diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index c19cef0c89..beae6ca8ad 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -16,12 +16,12 @@ package org.thingsboard.server.controller.plugin; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.BeanCreationNotAllowedException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.PongMessage; import org.springframework.web.socket.TextMessage; @@ -139,8 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr return; } var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - - internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsLimitQueuePerWsSession())); + internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsMsgQueueLimitPerSession())); externalSessionMap.put(externalSessionId, internalSessionId); processInWebSocketService(sessionRef, SessionEvent.onEstablished()); @@ -298,14 +297,13 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr String externalId = sessionRef.getSessionId(); log.debug("[{}] Processing {}", externalId, msg); String internalId = externalSessionMap.get(externalId); - - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - if (internalId != null) { SessionMetaData sessionMd = internalSessionMap.get(internalId); if (sessionMd != null) { - if (!StringUtils.isEmpty(tenantProfileConfiguration.getWsLimitUpdatesPerSession())) { - TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(tenantProfileConfiguration.getWsLimitUpdatesPerSession())); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); + if (StringUtils.isNotEmpty(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit())) { + // fixme: is this ok not to update rate limits config if it was change in profile? + TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit())); if (!rateLimits.tryConsume()) { if (blacklistedSessions.putIfAbsent(externalId, sessionRef) == null) { log.info("[{}][{}][{}] Failed to process session update. Max session updates limit reached" @@ -364,15 +362,15 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - if(tenantProfileConfiguration == null) { + if (tenantProfileConfiguration == null) { return true; } String sessionId = session.getId(); - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant() > 0) { + if (tenantProfileConfiguration.getMaxWsSessionsPerTenant() > 0) { Set tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSessions) { - if (tenantSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant()) { + if (tenantSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerTenant()) { tenantSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max tenant sessions limit reached" @@ -384,10 +382,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer() > 0) { + if (tenantProfileConfiguration.getMaxWsSessionsPerCustomer() > 0) { Set customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { - if (customerSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer()) { + if (customerSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerCustomer()) { customerSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max customer sessions limit reached" @@ -397,11 +395,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } } } - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser() > 0 + if (tenantProfileConfiguration.getMaxWsSessionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { - if (regularUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser()) { + if (regularUserSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerRegularUser()) { regularUserSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max regular user sessions limit reached" @@ -411,11 +409,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } } } - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser() > 0 + if (tenantProfileConfiguration.getMaxWsSessionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { - if (publicUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser()) { + if (publicUserSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerPublicUser()) { publicUserSessions.add(sessionId); } else { log.info("[{}][{}][{}] Failed to start session. Max public user sessions limit reached" @@ -435,26 +433,26 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr String sessionId = session.getId(); perSessionUpdateLimits.remove(sessionRef.getSessionId()); blacklistedSessions.remove(sessionRef.getSessionId()); - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerTenant() > 0) { + if (tenantProfileConfiguration.getMaxWsSessionsPerTenant() > 0) { Set tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSessions) { tenantSessions.remove(sessionId); } } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerCustomer() > 0) { + if (tenantProfileConfiguration.getMaxWsSessionsPerCustomer() > 0) { Set customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { customerSessions.remove(sessionId); } } - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSessionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { regularUserSessions.remove(sessionId); } } - if (tenantProfileConfiguration.getWsLimitMaxSessionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSessionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { publicUserSessions.remove(sessionId); diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 02bc87aaf9..d8e84458bf 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -216,6 +216,9 @@ public class ThingsboardInstallService { dataUpdateService.updateData("3.3.2"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); + case "3.3.3": + log.info("Upgrading ThingsBoard from version 3.3.3 to 3.4 ..."); + dataUpdateService.updateData("3.3.3"); break; //TODO update CacheCleanupService on the next version upgrade diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 7652114166..d23a8eecc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.flow.TbRuleChainInputNode; import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration; import org.thingsboard.rule.engine.profile.TbDeviceProfileNode; @@ -56,12 +55,9 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; -import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; -import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.device.DeviceProfileRepository; @@ -101,9 +97,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private TimeseriesService tsService; - @Autowired - private AlarmService alarmService; - @Autowired private EntityService entityService; @@ -113,9 +106,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private DeviceProfileRepository deviceProfileRepository; - @Autowired - private OAuth2Service oAuth2Service; - @Autowired private RateLimitsUpdater rateLimitsUpdater; @@ -140,12 +130,15 @@ public class DefaultDataUpdateService implements DataUpdateService { tenantsAlarmsCustomerUpdater.updateEntities(null); deviceProfileEntityDynamicConditionsUpdater.updateEntities(null); updateOAuth2Params(); - rateLimitsUpdater.updateEntities(null); break; case "3.3.2": log.info("Updating data from version 3.3.2 to 3.3.3 ..."); updateNestedRuleChains(); break; + case "3.3.3": + log.info("Updating data from version 3.3.3 to 3.4 ..."); + rateLimitsUpdater.updateEntities(); + break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java index c10d1b4bd7..1ec06a4413 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java @@ -50,6 +50,10 @@ public abstract class PaginatedUpdater { } } + public void updateEntities() { + updateEntities(null); + } + protected boolean forceReportTotal() { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index e333641da3..f66459c006 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -15,53 +15,53 @@ */ package org.thingsboard.server.service.install.update; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.tenant.TenantProfileService; @Component class RateLimitsUpdater extends PaginatedUpdater { + @Value("${server.rest.limits.tenant.enabled}") - boolean tenantServerRateLimitsEnabled; - @Value("${server.rest.limits.customer.enabled}") - boolean customerServerRateLimitsEnabled; + boolean tenantServerRestLimitsEnabled; @Value("${server.rest.limits.tenant.configuration}") - String tenantServerRateLimitsConfiguration; + String tenantServerRestLimitsConfiguration; + @Value("${server.rest.limits.customer.enabled}") + boolean customerServerRestLimitsEnabled; @Value("${server.rest.limits.customer.configuration}") - String customerServerRateLimitsConfiguration; + String customerServerRestLimitsConfiguration; @Value("${server.ws.limits.max_sessions_per_tenant}") - private int wsLimitMaxSessionsPerTenant; + private int maxWsSessionsPerTenant; @Value("${server.ws.limits.max_sessions_per_customer}") - private int wsLimitMaxSessionsPerCustomer; + private int maxWsSessionsPerCustomer; @Value("${server.ws.limits.max_sessions_per_regular_user}") - private int wsLimitMaxSessionsPerRegularUser; + private int maxWsSessionsPerRegularUser; @Value("${server.ws.limits.max_sessions_per_public_user}") - private int wsLimitMaxSessionsPerPublicUser; + private int maxWsSessionsPerPublicUser; @Value("${server.ws.limits.max_queue_per_ws_session}") - private int wsLimitQueuePerWsSession; + private int wsMsgQueueLimitPerSession; @Value("${server.ws.limits.max_subscriptions_per_tenant}") - private long wsLimitMaxSubscriptionsPerTenant; + private long maxWsSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer}") - private long wsLimitMaxSubscriptionsPerCustomer; + private long maxWsSubscriptionsPerCustomer; @Value("${server.ws.limits.max_subscriptions_per_regular_user}") - private long wsLimitMaxSubscriptionsPerRegularUser; + private long maxWsSubscriptionsPerRegularUser; @Value("${server.ws.limits.max_subscriptions_per_public_user}") - private long wsLimitMaxSubscriptionsPerPublicUser; + private long maxWsSubscriptionsPerPublicUser; @Value("${server.ws.limits.max_updates_per_session}") - private String wsLimitUpdatesPerSession; + private String wsUpdatesPerSessionRateLimit; @Value("${cassandra.query.tenant_rate_limits.enabled}") - private boolean cassandraLimitsIsEnabled; + private boolean cassandraQueryTenantRateLimitsEnabled; @Value("${cassandra.query.tenant_rate_limits.configuration}") - private String cassandraTenantLimitsConfiguration; - @Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") - private boolean printTenantNames; + private String cassandraQueryTenantRateLimitsConfiguration; @Autowired private TenantProfileService tenantProfileService; @@ -78,40 +78,38 @@ class RateLimitsUpdater extends PaginatedUpdater { @Override protected PageData findEntities(String id, PageLink pageLink) { - return tenantProfileService.findTenantProfiles(null, pageLink); + return tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, pageLink); } @Override - protected void updateEntity(TenantProfile entity) { - var profileConfiguration = entity.getDefaultTenantProfileConfiguration(); - if (profileConfiguration != null) { - if (tenantServerRateLimitsEnabled && StringUtils.isNotEmpty(tenantServerRateLimitsConfiguration)) { - profileConfiguration.setRateLimitsTenantConfiguration(tenantServerRateLimitsConfiguration); - } - if (customerServerRateLimitsEnabled && StringUtils.isNotEmpty(customerServerRateLimitsConfiguration)) { - profileConfiguration.setRateLimitsCustomerConfiguration(customerServerRateLimitsConfiguration); - } + protected void updateEntity(TenantProfile tenantProfile) { + var profileConfiguration = tenantProfile.getDefaultTenantProfileConfiguration(); - profileConfiguration.setWsLimitMaxSessionsPerTenant(wsLimitMaxSessionsPerTenant); - profileConfiguration.setWsLimitMaxSessionsPerCustomer(wsLimitMaxSessionsPerCustomer); - profileConfiguration.setWsLimitMaxSessionsPerPublicUser(wsLimitMaxSessionsPerPublicUser); - profileConfiguration.setWsLimitMaxSessionsPerRegularUser(wsLimitMaxSessionsPerRegularUser); - profileConfiguration.setWsLimitMaxSubscriptionsPerTenant(wsLimitMaxSubscriptionsPerTenant); - profileConfiguration.setWsLimitMaxSubscriptionsPerCustomer(wsLimitMaxSubscriptionsPerCustomer); - profileConfiguration.setWsLimitMaxSubscriptionsPerPublicUser(wsLimitMaxSubscriptionsPerPublicUser); - profileConfiguration.setWsLimitMaxSubscriptionsPerRegularUser(wsLimitMaxSubscriptionsPerRegularUser); - profileConfiguration.setWsLimitQueuePerWsSession(wsLimitQueuePerWsSession); - - if (StringUtils.isNotEmpty(wsLimitUpdatesPerSession)) { - profileConfiguration.setWsLimitUpdatesPerSession(wsLimitUpdatesPerSession); - } + if (tenantServerRestLimitsEnabled && StringUtils.isNotEmpty(tenantServerRestLimitsConfiguration)) { + profileConfiguration.setTenantServerRestLimitsConfiguration(tenantServerRestLimitsConfiguration); + } + if (customerServerRestLimitsEnabled && StringUtils.isNotEmpty(customerServerRestLimitsConfiguration)) { + profileConfiguration.setCustomerServerRestLimitsConfiguration(customerServerRestLimitsConfiguration); + } - if (cassandraLimitsIsEnabled) { - profileConfiguration.setCassandraTenantLimitsConfiguration(cassandraTenantLimitsConfiguration); - profileConfiguration.setPrintTenantNames(printTenantNames); - } + profileConfiguration.setMaxWsSessionsPerTenant(maxWsSessionsPerTenant); + profileConfiguration.setMaxWsSessionsPerCustomer(maxWsSessionsPerCustomer); + profileConfiguration.setMaxWsSessionsPerPublicUser(maxWsSessionsPerPublicUser); + profileConfiguration.setMaxWsSessionsPerRegularUser(maxWsSessionsPerRegularUser); + profileConfiguration.setMaxWsSubscriptionsPerTenant(maxWsSubscriptionsPerTenant); + profileConfiguration.setMaxWsSubscriptionsPerCustomer(maxWsSubscriptionsPerCustomer); + profileConfiguration.setMaxWsSubscriptionsPerPublicUser(maxWsSubscriptionsPerPublicUser); + profileConfiguration.setMaxWsSubscriptionsPerRegularUser(maxWsSubscriptionsPerRegularUser); + profileConfiguration.setWsMsgQueueLimitPerSession(wsMsgQueueLimitPerSession); + if (StringUtils.isNotEmpty(wsUpdatesPerSessionRateLimit)) { + profileConfiguration.setWsUpdatesPerSessionRateLimit(wsUpdatesPerSessionRateLimit); + } - tenantProfileService.saveTenantProfile(null, entity); + if (cassandraQueryTenantRateLimitsEnabled && StringUtils.isNotEmpty(cassandraQueryTenantRateLimitsConfiguration)) { + profileConfiguration.setCassandraQueryTenantRateLimitsConfiguration(cassandraQueryTenantRateLimitsConfiguration); } + + tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 81f945e620..b377eac8aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -24,6 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.web.socket.CloseStatus; @@ -304,29 +305,29 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void processSessionClose(TelemetryWebSocketSessionRef sessionRef) { var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - if(tenantProfileConfiguration != null) { + if (tenantProfileConfiguration != null) { String sessionId = "[" + sessionRef.getSessionId() + "]"; - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant() > 0) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerTenant() > 0) { Set tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSubscriptions) { tenantSubscriptions.removeIf(subId -> subId.startsWith(sessionId)); } } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer() > 0) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerCustomer() > 0) { Set customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { customerSessions.removeIf(subId -> subId.startsWith(sessionId)); } } - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { regularUserSessions.removeIf(subId -> subId.startsWith(sessionId)); } } - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { publicUserSessions.removeIf(subId -> subId.startsWith(sessionId)); @@ -341,12 +342,12 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi String subId = "[" + sessionRef.getSessionId() + "]:[" + cmd.getCmdId() + "]"; try { - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant() > 0) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerTenant() > 0) { Set tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet()); synchronized (tenantSubscriptions) { if (cmd.isUnsubscribe()) { tenantSubscriptions.remove(subId); - } else if (tenantSubscriptions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerTenant()) { + } else if (tenantSubscriptions.size() < tenantProfileConfiguration.getMaxWsSubscriptionsPerTenant()) { tenantSubscriptions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max tenant subscriptions limit reached" @@ -358,12 +359,12 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } if (sessionRef.getSecurityCtx().isCustomerUser()) { - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer() > 0) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerCustomer() > 0) { Set customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet()); synchronized (customerSessions) { if (cmd.isUnsubscribe()) { customerSessions.remove(subId); - } else if (customerSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerCustomer()) { + } else if (customerSessions.size() < tenantProfileConfiguration.getMaxWsSubscriptionsPerCustomer()) { customerSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max customer subscriptions limit reached" @@ -373,10 +374,10 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } } } - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (regularUserSessions) { - if (regularUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerRegularUser()) { + if (regularUserSessions.size() < tenantProfileConfiguration.getMaxWsSubscriptionsPerRegularUser()) { regularUserSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max regular user subscriptions limit reached" @@ -386,10 +387,10 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } } } - if (tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { + if (tenantProfileConfiguration.getMaxWsSubscriptionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) { Set publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet()); synchronized (publicUserSessions) { - if (publicUserSessions.size() < tenantProfileConfiguration.getWsLimitMaxSubscriptionsPerPublicUser()) { + if (publicUserSessions.size() < tenantProfileConfiguration.getMaxWsSubscriptionsPerPublicUser()) { publicUserSessions.add(subId); } else { log.info("[{}][{}][{}] Failed to start subscription. Max public user subscriptions limit reached" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index ae75512625..3d6669e870 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -83,7 +83,7 @@ public class TenantProfile extends SearchTextBased implements H @ApiModelProperty(position = 1, value = "JSON object with the tenant profile Id. " + "Specify this field to update the tenant profile. " + "Referencing non-existing tenant profile Id will cause error. " + - "Omit this field to create new tenant profile." ) + "Omit this field to create new tenant profile.") @Override public TenantProfileId getId() { return super.getId(); @@ -133,6 +133,7 @@ public class TenantProfile extends SearchTextBased implements H public TenantProfileData createDefaultTenantProfileData() { TenantProfileData tpd = new TenantProfileData(); tpd.setConfiguration(new DefaultTenantProfileConfiguration()); + this.profileData = tpd; return tpd; } @@ -147,11 +148,7 @@ public class TenantProfile extends SearchTextBased implements H @JsonIgnore public DefaultTenantProfileConfiguration getDefaultTenantProfileConfiguration() { - if(getProfileData().getConfiguration() != null && - getProfileData().getConfiguration().getType().equals(TenantProfileType.DEFAULT)) { - return (DefaultTenantProfileConfiguration) this.profileData.getConfiguration(); - } - return null; + return getProfileConfiguration().orElse(null); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index bcc5681e87..e1b97d7f5a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -46,22 +46,21 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; - private String rateLimitsTenantConfiguration; - private String rateLimitsCustomerConfiguration; + private String tenantServerRestLimitsConfiguration; + private String customerServerRestLimitsConfiguration; - private int wsLimitMaxSessionsPerTenant; - private int wsLimitMaxSessionsPerCustomer; - private int wsLimitMaxSessionsPerRegularUser; - private int wsLimitMaxSessionsPerPublicUser; - private int wsLimitQueuePerWsSession; - private long wsLimitMaxSubscriptionsPerTenant; - private long wsLimitMaxSubscriptionsPerCustomer; - private long wsLimitMaxSubscriptionsPerRegularUser; - private long wsLimitMaxSubscriptionsPerPublicUser; - private String wsLimitUpdatesPerSession; + private int maxWsSessionsPerTenant; + private int maxWsSessionsPerCustomer; + private int maxWsSessionsPerRegularUser; + private int maxWsSessionsPerPublicUser; + private int wsMsgQueueLimitPerSession; + private long maxWsSubscriptionsPerTenant; + private long maxWsSubscriptionsPerCustomer; + private long maxWsSubscriptionsPerRegularUser; + private long maxWsSubscriptionsPerPublicUser; + private String wsUpdatesPerSessionRateLimit; - private String cassandraTenantLimitsConfiguration; - private boolean printTenantNames; + private String cassandraQueryTenantRateLimitsConfiguration; private long maxTransportMessages; private long maxTransportDataPoints; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index 80fbaea65f..9e435ea34e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -29,11 +29,10 @@ import java.time.Duration; public class TbRateLimits { private final LocalBucket bucket; @Getter - private final String currentConfig; + private final String configuration; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); - currentConfig = limitsConfiguration; boolean initialized = false; for (String limitSrc : limitsConfiguration.split(",")) { long capacity = Long.parseLong(limitSrc.split(":")[0]); @@ -46,8 +45,7 @@ public class TbRateLimits { } else { throw new IllegalArgumentException("Failed to parse rate limits configuration: " + limitsConfiguration); } - - + this.configuration = limitsConfiguration; } public boolean tryConsume() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateReadExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateReadExecutor.java index 54a9953887..70e89df502 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateReadExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateReadExecutor.java @@ -22,9 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; @@ -33,7 +31,6 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor; import org.thingsboard.server.dao.util.AsyncTaskContext; import org.thingsboard.server.dao.util.NoSqlAnyDao; -import org.thingsboard.server.dao.util.TenantRateLimitException; import javax.annotation.PreDestroy; @@ -45,13 +42,6 @@ import javax.annotation.PreDestroy; @NoSqlAnyDao public class CassandraBufferedRateReadExecutor extends AbstractBufferedRateExecutor { - @Autowired - private EntityService entityService; - @Autowired - private TbTenantProfileCache tenantProfileCache; - - private Map tenantNamesCache = new HashMap<>(); - static final String BUFFER_NAME = "Read"; public CassandraBufferedRateReadExecutor( @@ -64,9 +54,10 @@ public class CassandraBufferedRateReadExecutor extends AbstractBufferedRateExecu @Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") boolean printTenantNames, @Value("${cassandra.query.print_queries_freq:0}") int printQueriesFreq, @Autowired StatsFactory statsFactory, - @Autowired EntityService entityService) { + @Autowired EntityService entityService, + @Autowired TbTenantProfileCache tenantProfileCache) { super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, printQueriesFreq, statsFactory, - entityService, printTenantNames); + entityService, tenantProfileCache, printTenantNames); } @Scheduled(fixedDelayString = "${cassandra.query.rate_limit_print_interval_ms}") @@ -104,24 +95,4 @@ public class CassandraBufferedRateReadExecutor extends AbstractBufferedRateExecu ); } - @Override - protected boolean checkRateLimits(CassandraStatementTask task, SettableFuture future) { - var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultTenantProfileConfiguration(); - if (StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraTenantLimitsConfiguration())) { - if (task.getTenantId() == null) { - log.info("Invalid task received: {}", task); - } else if (!task.getTenantId().isNullUid()) { - TbRateLimits rateLimits = perTenantLimits.computeIfAbsent( - task.getTenantId(), id -> new TbRateLimits(tenantProfileConfiguration.getCassandraTenantLimitsConfiguration()) - ); - if (!rateLimits.tryConsume()) { - stats.incrementRateLimitedTenant(task.getTenantId()); - stats.getTotalRateLimited().increment(); - future.setException(new TenantRateLimitException()); - return true; - } - } - } - return false; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateWriteExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateWriteExecutor.java index 990baa1e2a..ebcf9cf38f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateWriteExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateWriteExecutor.java @@ -24,6 +24,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor; import org.thingsboard.server.dao.util.AsyncTaskContext; import org.thingsboard.server.dao.util.NoSqlAnyDao; @@ -47,14 +48,13 @@ public class CassandraBufferedRateWriteExecutor extends AbstractBufferedRateExec @Value("${cassandra.query.dispatcher_threads:2}") int dispatcherThreads, @Value("${cassandra.query.callback_threads:4}") int callbackThreads, @Value("${cassandra.query.poll_ms:50}") long pollMs, - @Value("${cassandra.query.tenant_rate_limits.enabled}") boolean tenantRateLimitsEnabled, - @Value("${cassandra.query.tenant_rate_limits.configuration}") String tenantRateLimitsConfiguration, @Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") boolean printTenantNames, @Value("${cassandra.query.print_queries_freq:0}") int printQueriesFreq, @Autowired StatsFactory statsFactory, - @Autowired EntityService entityService) { - super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, tenantRateLimitsEnabled, tenantRateLimitsConfiguration, printQueriesFreq, statsFactory, - entityService, printTenantNames); + @Autowired EntityService entityService, + @Autowired TbTenantProfileCache tenantProfileCache) { + super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, printQueriesFreq, statsFactory, + entityService, tenantProfileCache, printTenantNames); } @Scheduled(fixedDelayString = "${cassandra.query.rate_limit_print_interval_ms}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index 76710f7fab..b478be0d7b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -30,6 +30,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.stats.DefaultCounter; @@ -38,6 +39,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.nosql.CassandraStatementTask; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import javax.annotation.Nullable; import java.util.HashMap; @@ -71,22 +73,22 @@ public abstract class AbstractBufferedRateExecutor perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private final AtomicInteger printQueriesIdx = new AtomicInteger(0); protected final AtomicInteger concurrencyLevel; protected final BufferedRateExecutorStats stats; - private final EntityService entityService; - private final Map tenantNamesCache = new HashMap<>(); + private final TbTenantProfileCache tenantProfileCache; private final boolean printTenantNames; + private final Map tenantNamesCache = new HashMap<>(); public AbstractBufferedRateExecutor(int queueLimit, int concurrencyLimit, long maxWaitTime, int dispatcherThreads, int callbackThreads, long pollMs, int printQueriesFreq, StatsFactory statsFactory, - EntityService entityService, boolean printTenantNames) { + EntityService entityService, TbTenantProfileCache tenantProfileCache, boolean printTenantNames) { this.maxWaitTime = maxWaitTime; this.pollMs = pollMs; this.concurrencyLimit = concurrencyLimit; @@ -100,6 +102,7 @@ public abstract class AbstractBufferedRateExecutor future); - @Override public F submit(T task) { SettableFuture settableFuture = create(); F result = wrap(task, settableFuture); - boolean perTenantLimitReached = checkRateLimits(task, settableFuture); + + boolean perTenantLimitReached = false; + var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultTenantProfileConfiguration(); + if (StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraQueryTenantRateLimitsConfiguration())) { + if (task.getTenantId() == null) { + log.info("Invalid task received: {}", task); + } else if (!task.getTenantId().isNullUid()) { + TbRateLimits rateLimits = perTenantLimits.computeIfAbsent( + task.getTenantId(), id -> new TbRateLimits(tenantProfileConfiguration.getCassandraQueryTenantRateLimitsConfiguration()) + ); + if (!rateLimits.tryConsume()) { + stats.incrementRateLimitedTenant(task.getTenantId()); + stats.getTotalRateLimited().increment(); + settableFuture.setException(new TenantRateLimitException()); + perTenantLimitReached = true; + } + } + } if (!perTenantLimitReached) { try { From ffd2a02742673e8bd32951d6de5d3a4e3989517a Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 21 Jan 2022 11:37:53 +0200 Subject: [PATCH 017/262] Tenant profile rate limits configuration UI refactoring --- .../profile/tenant-profile.component.html | 8 +- ...enant-profile-configuration.component.html | 709 +++++++++--------- ...-tenant-profile-configuration.component.ts | 29 +- ui-ngx/src/app/shared/models/tenant.model.ts | 61 +- .../assets/locale/locale.constant-en_US.json | 31 +- 5 files changed, 403 insertions(+), 435 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html index e5a3a54293..18dd97fac4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html @@ -62,14 +62,14 @@
{{'tenant.isolated-tb-rule-engine-details' | translate}}
- - tenant-profile.description + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 2a975479bd..9ea904d820 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -15,87 +15,81 @@ limitations under the License. --> -
- - - - Thingsboard rate limits - - - - tenant-profile.maximum-devices - - - {{ 'tenant-profile.maximum-devices-required' | translate}} - - - {{ 'tenant-profile.maximum-devices-range' | translate}} - - - - tenant-profile.maximum-assets - - - {{ 'tenant-profile.maximum-assets-required' | translate}} - - - {{ 'tenant-profile.maximum-assets-range' | translate}} - - - - tenant-profile.maximum-customers - - - {{ 'tenant-profile.maximum-customers-required' | translate}} - - - {{ 'tenant-profile.maximum-customers-range' | translate}} - - - - tenant-profile.maximum-users - - - {{ 'tenant-profile.maximum-users-required' | translate}} - - - {{ 'tenant-profile.maximum-users-range' | translate}} - - - - tenant-profile.maximum-dashboards - - - {{ 'tenant-profile.maximum-dashboards-required' | translate}} - - - {{ 'tenant-profile.maximum-dashboards-range' | translate}} - - - - tenant-profile.maximum-rule-chains - - - {{ 'tenant-profile.maximum-rule-chains-required' | translate}} - - - {{ 'tenant-profile.maximum-rule-chains-range' | translate}} - - - - tenant-profile.maximum-resources-sum-data-size +
+ + tenant-profile.maximum-devices + + + {{ 'tenant-profile.maximum-devices-required' | translate}} + + + {{ 'tenant-profile.maximum-devices-range' | translate}} + + + + tenant-profile.maximum-assets + + + {{ 'tenant-profile.maximum-assets-required' | translate}} + + + {{ 'tenant-profile.maximum-assets-range' | translate}} + + + + tenant-profile.maximum-customers + + + {{ 'tenant-profile.maximum-customers-required' | translate}} + + + {{ 'tenant-profile.maximum-customers-range' | translate}} + + + + tenant-profile.maximum-users + + + {{ 'tenant-profile.maximum-users-required' | translate}} + + + {{ 'tenant-profile.maximum-users-range' | translate}} + + + + tenant-profile.maximum-dashboards + + + {{ 'tenant-profile.maximum-dashboards-required' | translate}} + + + {{ 'tenant-profile.maximum-dashboards-range' | translate}} + + + + tenant-profile.maximum-rule-chains + + + {{ 'tenant-profile.maximum-rule-chains-required' | translate}} + + + {{ 'tenant-profile.maximum-rule-chains-range' | translate}} + + + + tenant-profile.maximum-resources-sum-data-size @@ -120,74 +114,74 @@ tenant-profile.max-transport-messages - - - {{ 'tenant-profile.max-transport-messages-range' | translate}} - - - {{ 'tenant-profile.max-transport-messages-required' | translate}} - - - - tenant-profile.max-transport-data-points - - - {{ 'tenant-profile.max-transport-data-points-required' | translate}} - - - {{ 'tenant-profile.max-transport-data-points-range' | translate}} - - - - tenant-profile.max-r-e-executions - - - {{ 'tenant-profile.max-r-e-executions-required' | translate}} - - - {{ 'tenant-profile.max-r-e-executions-range' | translate}} - - - - tenant-profile.max-j-s-executions - - - {{ 'tenant-profile.max-j-s-executions-required' | translate}} - - - {{ 'tenant-profile.max-j-s-executions-range' | translate}} - - - - tenant-profile.max-d-p-storage-days - - - {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} - - - {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} - - - - tenant-profile.default-storage-ttl-days - - - {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} - - - {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-required' | translate}} + + + + tenant-profile.max-transport-data-points + + + {{ 'tenant-profile.max-transport-data-points-required' | translate}} + + + {{ 'tenant-profile.max-transport-data-points-range' | translate}} + + + + tenant-profile.max-r-e-executions + + + {{ 'tenant-profile.max-r-e-executions-required' | translate}} + + + {{ 'tenant-profile.max-r-e-executions-range' | translate}} + + + + tenant-profile.max-j-s-executions + + + {{ 'tenant-profile.max-j-s-executions-required' | translate}} + + + {{ 'tenant-profile.max-j-s-executions-range' | translate}} + + + + tenant-profile.max-d-p-storage-days + + + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} + + + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} + + + + tenant-profile.default-storage-ttl-days + + + {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} + + + {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} @@ -212,222 +206,205 @@ {{ 'tenant-profile.rpc-ttl-days-days-range' | translate}} - - - - tenant-profile.max-rule-node-executions-per-message - - - {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} - - - {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} - - - - tenant-profile.max-emails - - - {{ 'tenant-profile.max-emails-required' | translate}} - - - {{ 'tenant-profile.max-emails-range' | translate}} - - - - tenant-profile.max-sms - - - {{ 'tenant-profile.max-sms-required' | translate}} - - - {{ 'tenant-profile.max-sms-range' | translate}} - - - - tenant-profile.max-created-alarms - - - {{ 'tenant-profile.max-created-alarms-required' | translate}} - - - {{ 'tenant-profile.max-created-alarms-range' | translate}} - - - - - - - - Server rate limits - - - - tenant-profile.tenant-rest-limits - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.customer-rest-limits - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - - - - - Transport rate limits - - - - tenant-profile.transport-tenant-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-tenant-telemetry-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-tenant-telemetry-data-points-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-device-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-device-telemetry-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-device-telemetry-data-points-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - - - - - Web Socket rate limits - - - - tenant-profile.ws-limit-max-sessions-per-tenant - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-customer - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-public-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-queue-per-session - - - {{ 'tenant-profile.too-small-value-one' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-tenant - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-customer - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-regular-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-public-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-updates-per-session - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - + + + + tenant-profile.max-rule-node-executions-per-message + + + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} + + + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} + + + + tenant-profile.max-emails + + + {{ 'tenant-profile.max-emails-required' | translate}} + + + {{ 'tenant-profile.max-emails-range' | translate}} + + + + tenant-profile.max-sms + + + {{ 'tenant-profile.max-sms-required' | translate}} + + + {{ 'tenant-profile.max-sms-range' | translate}} + + + + tenant-profile.max-created-alarms + + + {{ 'tenant-profile.max-created-alarms-required' | translate}} + + + {{ 'tenant-profile.max-created-alarms-range' | translate}} + + - - - Cassandra rate limits - - - - tenant-profile.cassandra-tenant-limits-configuration - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - {{ 'tenant-profile.cassandra-print-tenant-names' | translate }} - - - - + + tenant-profile.transport-tenant-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-tenant-telemetry-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-tenant-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-telemetry-msg-rate-limit + + + + tenant-profile.transport-device-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.tenant-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.customer-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-sessions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-queue-per-session + + + {{ 'tenant-profile.too-small-value-one' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-regular-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-updates-per-session + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.cassandra-tenant-limits-configuration + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index f9fe3bc3b1..8bcde9a68b 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -80,22 +80,19 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA defaultStorageTtlDays: [null, [Validators.required, Validators.min(0)]], alarmsTtlDays: [null, [Validators.required, Validators.min(0)]], rpcTtlDays: [null, [Validators.required, Validators.min(0)]], - - rateLimitsTenantConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], - rateLimitsCustomerConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], - - wsLimitMaxSessionsPerTenant: [null, [Validators.min(0)]], - wsLimitMaxSessionsPerCustomer: [null, [Validators.min(0)]], - wsLimitMaxSessionsPerPublicUser: [null, [Validators.min(0)]], - wsLimitQueuePerWsSession: [null, [Validators.min(0)]], - wsLimitMaxSubscriptionsPerTenant: [null, [Validators.min(0)]], - wsLimitMaxSubscriptionsPerCustomer: [null, [Validators.min(0)]], - wsLimitMaxSubscriptionsPerRegularUser: [null, [Validators.min(0)]], - wsLimitMaxSubscriptionsPerPublicUser: [null, [Validators.min(0)]], - wsLimitUpdatesPerSession: [null, [Validators.pattern(this.rateLimitsPattern)]], - - cassandraTenantLimitsConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], - printTenantNames: [null, []] + tenantServerRestLimitsConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], + customerServerRestLimitsConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]], + maxWsSessionsPerTenant: [null, [Validators.min(0)]], + maxWsSessionsPerCustomer: [null, [Validators.min(0)]], + maxWsSessionsPerRegularUser: [null, [Validators.min(0)]], + maxWsSessionsPerPublicUser: [null, [Validators.min(0)]], + wsMsgQueueLimitPerSession: [null, [Validators.min(0)]], + maxWsSubscriptionsPerTenant: [null, [Validators.min(0)]], + maxWsSubscriptionsPerCustomer: [null, [Validators.min(0)]], + maxWsSubscriptionsPerRegularUser: [null, [Validators.min(0)]], + maxWsSubscriptionsPerPublicUser: [null, [Validators.min(0)]], + wsUpdatesPerSessionRateLimit: [null, [Validators.pattern(this.rateLimitsPattern)]], + cassandraQueryTenantRateLimitsConfiguration: [null, [Validators.pattern(this.rateLimitsPattern)]] }); this.defaultTenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { this.updateModel(); diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 910e48d0a6..26b5a0aa88 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -51,22 +51,21 @@ export interface DefaultTenantProfileConfiguration { maxSms: number; maxCreatedAlarms: number; - rateLimitsTenantConfiguration: string; - rateLimitsCustomerConfiguration: string; - - wsLimitMaxSessionsPerTenant: number; - wsLimitMaxSessionsPerCustomer: number; - wsLimitMaxSessionsPerRegularUser: number; - wsLimitMaxSessionsPerPublicUser: number; - wsLimitQueuePerWsSession: number; - wsLimitMaxSubscriptionsPerTenant: number; - wsLimitMaxSubscriptionsPerCustomer: number; - wsLimitMaxSubscriptionsPerRegularUser: number; - wsLimitMaxSubscriptionsPerPublicUser: number; - wsLimitUpdatesPerSession: string; - - cassandraTenantLimitsConfiguration: string; - printTenantNames: boolean; + tenantServerRestLimitsConfiguration: string; + customerServerRestLimitsConfiguration: string; + + maxWsSessionsPerTenant: number; + maxWsSessionsPerCustomer: number; + maxWsSessionsPerRegularUser: number; + maxWsSessionsPerPublicUser: number; + wsMsgQueueLimitPerSession: number; + maxWsSubscriptionsPerTenant: number; + maxWsSubscriptionsPerCustomer: number; + maxWsSubscriptionsPerRegularUser: number; + maxWsSubscriptionsPerPublicUser: number; + wsUpdatesPerSessionRateLimit: string; + + cassandraQueryTenantRateLimitsConfiguration: string; defaultStorageTtlDays: number; alarmsTtlDays: number; @@ -102,26 +101,22 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxEmails: 0, maxSms: 0, maxCreatedAlarms: 0, + tenantServerRestLimitsConfiguration: '', + customerServerRestLimitsConfiguration: '', + maxWsSessionsPerTenant: 0, + maxWsSessionsPerCustomer: 0, + maxWsSessionsPerRegularUser: 0, + maxWsSessionsPerPublicUser: 0, + wsMsgQueueLimitPerSession: 0, + maxWsSubscriptionsPerTenant: 0, + maxWsSubscriptionsPerCustomer: 0, + maxWsSubscriptionsPerRegularUser: 0, + maxWsSubscriptionsPerPublicUser: 0, + wsUpdatesPerSessionRateLimit: '', + cassandraQueryTenantRateLimitsConfiguration: '', defaultStorageTtlDays: 0, alarmsTtlDays: 0, rpcTtlDays: 0, - - rateLimitsTenantConfiguration: '', - rateLimitsCustomerConfiguration: '', - - wsLimitMaxSessionsPerTenant: 0, - wsLimitMaxSessionsPerCustomer: 0, - wsLimitMaxSessionsPerRegularUser: 0, - wsLimitMaxSessionsPerPublicUser: 0, - wsLimitQueuePerWsSession: 500, - wsLimitMaxSubscriptionsPerTenant: 0, - wsLimitMaxSubscriptionsPerCustomer: 0, - wsLimitMaxSubscriptionsPerRegularUser: 0, - wsLimitMaxSubscriptionsPerPublicUser: 0, - wsLimitUpdatesPerSession: '' , - - cassandraTenantLimitsConfiguration: '', - printTenantNames: false }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c5e49e64f0..851e8933f7 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2814,22 +2814,21 @@ "max-created-alarms": "Maximum number of alarms created (0 - unlimited)", "max-created-alarms-required": "Maximum number of alarms created is required.", "max-created-alarms-range": "Maximum number of alarms created can't be negative", - "tenant-rest-limits": "Server limits of request for Tenant", - "customer-rest-limits": "Server limits of request for Customer", - "incorrect-pattern-for-rate-limits": "Here should be conformity of a number of requests per seconds divided by colon, comma separated. For example: 100:1,2000:60", - "too-small-value-zero": "Too small value specified, it should be more than 0", - "too-small-value-one": "Too small value specified, it should be more than 1", - "cassandra-tenant-limits-configuration": "Tenant limits configuration", - "ws-limit-max-sessions-per-tenant": "Max sessions per Tenant", - "ws-limit-max-sessions-per-customer": "Max sessions per Customer", - "ws-limit-max-sessions-per-public-user": "Max sessions per Public User", - "ws-limit-queue-per-session": "Limit queue per session", - "ws-limit-max-subscriptions-per-tenant": "Max subscriptions per Tenant", - "ws-limit-max-subscriptions-per-customer": "Max subscriptions per Customer", - "ws-limit-max-subscriptions-per-regular-user": "Max subscriptions per regular User", - "ws-limit-max-subscriptions-per-public-user": "Max subscriptions per public User", - "ws-limit-updates-per-session": "Constraint of updates per session", - "cassandra-print-tenant-names": "Print Tenant names to log" + "tenant-rest-limits": "Rate limit for REST requests for tenant", + "customer-rest-limits": "Rate limit for REST requests for customer", + "incorrect-pattern-for-rate-limits": "The format is comma separated pairs of capacity and period (in seconds) with a colon between, e.g. 100:1,2000:60", + "too-small-value-zero": "The value must be bigger than 0", + "too-small-value-one": "The value must be bigger than 1", + "cassandra-tenant-limits-configuration": "Cassandra query rate limit for tenant", + "ws-limit-max-sessions-per-tenant": "Maximum number of WS sessions per tenant", + "ws-limit-max-sessions-per-customer": "Maximum number of WS sessions per customer", + "ws-limit-max-sessions-per-public-user": "Maximum number of WS sessions per public user", + "ws-limit-queue-per-session": "Maximum size of WS message queue per session", + "ws-limit-max-subscriptions-per-tenant": "Maximum number of WS subscriptions per tenant", + "ws-limit-max-subscriptions-per-customer": "Maximum number of WS subscriptions per customer", + "ws-limit-max-subscriptions-per-regular-user": "Maximum number of WS subscriptions per regular user", + "ws-limit-max-subscriptions-per-public-user": "Maximum number of WS subscriptions per public user", + "ws-limit-updates-per-session": "Rate limit for WS updates per session" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", From d99a8066fe6a756d02edd2e262c2101736de67d6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 21 Jan 2022 15:18:39 +0200 Subject: [PATCH 018/262] Refactor RateLimitsUpdater; handle rate limits disabling --- .../config/RateLimitProcessingFilter.java | 56 ++++++++++--------- .../controller/plugin/TbWebSocketHandler.java | 16 +++--- .../install/update/RateLimitsUpdater.java | 34 +++++------ .../DefaultTelemetryWebSocketService.java | 4 +- .../src/main/resources/thingsboard.yml | 21 ------- .../server/common/data/TenantProfile.java | 10 ++-- .../DefaultTenantProfileConfiguration.java | 20 +++---- .../util/AbstractBufferedRateExecutor.java | 4 +- 8 files changed, 74 insertions(+), 91 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 7163ba3feb..3c97164991 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -15,14 +15,14 @@ */ package org.thingsboard.server.config; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import org.springframework.web.filter.GenericFilterBean; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; @@ -36,6 +36,7 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -55,37 +56,38 @@ public class RateLimitProcessingFilter extends GenericFilterBean { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultTenantProfileConfiguration(); - if (profileConfiguration != null) { - if (user.isTenantAdmin() && StringUtils.isNotEmpty(profileConfiguration.getTenantServerRestLimitsConfiguration())) { - TbRateLimits rateLimits = perTenantLimits.get(user.getTenantId()); - if (rateLimits == null || !rateLimits.getConfiguration().equals(profileConfiguration.getTenantServerRestLimitsConfiguration())) { - // fixme: or maybe handle component lifecycle event ? - rateLimits = new TbRateLimits(profileConfiguration.getTenantServerRestLimitsConfiguration()); - perTenantLimits.put(user.getTenantId(), rateLimits); - } - - if (!rateLimits.tryConsume()) { - errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response); - return; - } - } else if (user.isCustomerUser() && StringUtils.isNotEmpty(profileConfiguration.getCustomerServerRestLimitsConfiguration())) { - TbRateLimits rateLimits = perCustomerLimits.get(user.getCustomerId()); - if (rateLimits == null || !rateLimits.getConfiguration().equals(profileConfiguration.getCustomerServerRestLimitsConfiguration())) { - rateLimits = new TbRateLimits(profileConfiguration.getCustomerServerRestLimitsConfiguration()); - perCustomerLimits.put(user.getCustomerId(), rateLimits); - } - - if (!rateLimits.tryConsume()) { - errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response); - return; - } + var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultProfileConfiguration(); + if (!checkRateLimits(user.getTenantId(), profileConfiguration.getTenantServerRestLimitsConfiguration(), perTenantLimits, response)) { + return; + } + if (user.isCustomerUser()) { + if (!checkRateLimits(user.getCustomerId(), profileConfiguration.getCustomerServerRestLimitsConfiguration(), perCustomerLimits, response)) { + return; } } } chain.doFilter(request, response); } + private boolean checkRateLimits(I ownerId, String rateLimitConfig, Map rateLimitsMap, ServletResponse response) { + if (StringUtils.isNotEmpty(rateLimitConfig)) { + TbRateLimits rateLimits = rateLimitsMap.get(ownerId); + if (rateLimits == null || !rateLimits.getConfiguration().equals(rateLimitConfig)) { + rateLimits = new TbRateLimits(rateLimitConfig); + rateLimitsMap.put(ownerId, rateLimits); + } + + if (!rateLimits.tryConsume()) { + errorResponseHandler.handle(new TbRateLimitsException(ownerId.getEntityType()), (HttpServletResponse) response); + return false; + } + } else { + rateLimitsMap.remove(ownerId); + } + + return true; + } + protected SecurityUser getCurrentUser() { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); if (authentication != null && authentication.getPrincipal() instanceof SecurityUser) { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index beae6ca8ad..cbbfd407af 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -38,7 +38,6 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; -import org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocketService; import org.thingsboard.server.service.telemetry.SessionEvent; import org.thingsboard.server.service.telemetry.TelemetryWebSocketMsgEndpoint; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; @@ -138,8 +137,9 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (!checkLimits(session, sessionRef)) { return; } - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsMsgQueueLimitPerSession())); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); + internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsMsgQueueLimitPerSession() > 0 ? + tenantProfileConfiguration.getWsMsgQueueLimitPerSession() : 500)); externalSessionMap.put(externalSessionId, internalSessionId); processInWebSocketService(sessionRef, SessionEvent.onEstablished()); @@ -300,9 +300,8 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (internalId != null) { SessionMetaData sessionMd = internalSessionMap.get(internalId); if (sessionMd != null) { - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); if (StringUtils.isNotEmpty(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit())) { - // fixme: is this ok not to update rate limits config if it was change in profile? TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit())); if (!rateLimits.tryConsume()) { if (blacklistedSessions.putIfAbsent(externalId, sessionRef) == null) { @@ -315,6 +314,8 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr log.debug("[{}][{}][{}] Session is no longer blacklisted.", sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), externalId); blacklistedSessions.remove(externalId); } + } else { + perSessionUpdateLimits.remove(sessionRef.getSessionId()); } sessionMd.sendMsg(msg); } else { @@ -360,8 +361,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr private boolean checkLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) throws Exception { var tenantProfileConfiguration = - tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); - + tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); if (tenantProfileConfiguration == null) { return true; } @@ -428,7 +428,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr } private void cleanupLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) { - var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); String sessionId = session.getId(); perSessionUpdateLimits.remove(sessionRef.getSessionId()); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index f66459c006..054f68f71d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -28,39 +28,39 @@ import org.thingsboard.server.dao.tenant.TenantProfileService; @Component class RateLimitsUpdater extends PaginatedUpdater { - @Value("${server.rest.limits.tenant.enabled}") + @Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_TENANT_ENABLED') ?: environment.getProperty('server.rest.limits.tenant.enabled') ?: 'false' }") boolean tenantServerRestLimitsEnabled; - @Value("${server.rest.limits.tenant.configuration}") + @Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_TENANT_CONFIGURATION') ?: environment.getProperty('server.rest.limits.tenant.configuration') ?: '100:1,2000:60' }") String tenantServerRestLimitsConfiguration; - @Value("${server.rest.limits.customer.enabled}") + @Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_CUSTOMER_ENABLED') ?: environment.getProperty('server.rest.limits.customer.enabled') ?: 'false' }") boolean customerServerRestLimitsEnabled; - @Value("${server.rest.limits.customer.configuration}") + @Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_CUSTOMER_CONFIGURATION') ?: environment.getProperty('server.rest.limits.customer.configuration') ?: '50:1,1000:60' }") String customerServerRestLimitsConfiguration; - @Value("${server.ws.limits.max_sessions_per_tenant}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_TENANT') ?: environment.getProperty('server.ws.limits.max_sessions_per_tenant') ?: '0' }") private int maxWsSessionsPerTenant; - @Value("${server.ws.limits.max_sessions_per_customer}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_CUSTOMER') ?: environment.getProperty('server.ws.limits.max_sessions_per_customer') ?: '0' }") private int maxWsSessionsPerCustomer; - @Value("${server.ws.limits.max_sessions_per_regular_user}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_REGULAR_USER') ?: environment.getProperty('server.ws.limits.max_sessions_per_regular_user') ?: '0' }") private int maxWsSessionsPerRegularUser; - @Value("${server.ws.limits.max_sessions_per_public_user}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_PUBLIC_USER') ?: environment.getProperty('server.ws.limits.max_sessions_per_public_user') ?: '0' }") private int maxWsSessionsPerPublicUser; - @Value("${server.ws.limits.max_queue_per_ws_session}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_QUEUE_PER_WS_SESSION') ?: environment.getProperty('server.ws.limits.max_queue_per_ws_session') ?: '500' }") private int wsMsgQueueLimitPerSession; - @Value("${server.ws.limits.max_subscriptions_per_tenant}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_TENANT') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_tenant') ?: '0' }") private long maxWsSubscriptionsPerTenant; - @Value("${server.ws.limits.max_subscriptions_per_customer}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_CUSTOMER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_customer') ?: '0' }") private long maxWsSubscriptionsPerCustomer; - @Value("${server.ws.limits.max_subscriptions_per_regular_user}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_REGULAR_USER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_regular_user') ?: '0' }") private long maxWsSubscriptionsPerRegularUser; - @Value("${server.ws.limits.max_subscriptions_per_public_user}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_PUBLIC_USER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_public_user') ?: '0' }") private long maxWsSubscriptionsPerPublicUser; - @Value("${server.ws.limits.max_updates_per_session}") + @Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_UPDATES_PER_SESSION') ?: environment.getProperty('server.ws.limits.max_updates_per_session') ?: '300:1,3000:60' }") private String wsUpdatesPerSessionRateLimit; - @Value("${cassandra.query.tenant_rate_limits.enabled}") + @Value("#{ environment.getProperty('CASSANDRA_QUERY_TENANT_RATE_LIMITS_ENABLED') ?: environment.getProperty('cassandra.query.tenant_rate_limits.enabled') ?: 'false' }") private boolean cassandraQueryTenantRateLimitsEnabled; - @Value("${cassandra.query.tenant_rate_limits.configuration}") + @Value("#{ environment.getProperty('CASSANDRA_QUERY_TENANT_RATE_LIMITS_CONFIGURATION') ?: environment.getProperty('cassandra.query.tenant_rate_limits.configuration') ?: '1000:1,30000:60' }") private String cassandraQueryTenantRateLimitsConfiguration; @Autowired @@ -83,7 +83,7 @@ class RateLimitsUpdater extends PaginatedUpdater { @Override protected void updateEntity(TenantProfile tenantProfile) { - var profileConfiguration = tenantProfile.getDefaultTenantProfileConfiguration(); + var profileConfiguration = tenantProfile.getDefaultProfileConfiguration(); if (tenantServerRestLimitsEnabled && StringUtils.isNotEmpty(tenantServerRestLimitsConfiguration)) { profileConfiguration.setTenantServerRestLimitsConfiguration(tenantServerRestLimitsConfiguration); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index b377eac8aa..4af12a15af 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -304,7 +304,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } private void processSessionClose(TelemetryWebSocketSessionRef sessionRef) { - var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); + var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); if (tenantProfileConfiguration != null) { String sessionId = "[" + sessionRef.getSessionId() + "]"; @@ -338,7 +338,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } private boolean processSubscription(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd) { - var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultTenantProfileConfiguration(); + var tenantProfileConfiguration = (DefaultTenantProfileConfiguration) tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration(); String subId = "[" + sessionRef.getSessionId() + "]:[" + cmd.getCmdId() + "]"; try { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a502afb297..b24be9e2c3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -56,18 +56,6 @@ server: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" # recommended timeout >= 30 seconds. Platform will attempt to send 'ping' request 3 times within the timeout ping_timeout: "${TB_SERVER_WS_PING_TIMEOUT:30000}" - limits: - # Limit the amount of sessions and subscriptions available on each server. Put values to zero to disable particular limitation - max_sessions_per_tenant: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_TENANT:0}" - max_sessions_per_customer: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_CUSTOMER:0}" - max_sessions_per_regular_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_REGULAR_USER:0}" - max_sessions_per_public_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_PUBLIC_USER:0}" - max_queue_per_ws_session: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_QUEUE_PER_WS_SESSION:500}" - max_subscriptions_per_tenant: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_TENANT:0}" - max_subscriptions_per_customer: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_CUSTOMER:0}" - max_subscriptions_per_regular_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_REGULAR_USER:0}" - max_subscriptions_per_public_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_PUBLIC_USER:0}" - max_updates_per_session: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_UPDATES_PER_SESSION:300:1,3000:60}" dynamic_page_link: refresh_interval: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_INTERVAL_SEC:60}" refresh_pool_size: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_POOL_SIZE:1}" @@ -76,13 +64,6 @@ server: max_entities_per_data_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_DATA_SUBSCRIPTION:10000}" max_entities_per_alarm_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_ALARM_SUBSCRIPTION:10000}" rest: - limits: - tenant: - enabled: "${TB_SERVER_REST_LIMITS_TENANT_ENABLED:false}" - configuration: "${TB_SERVER_REST_LIMITS_TENANT_CONFIGURATION:100:1,2000:60}" - customer: - enabled: "${TB_SERVER_REST_LIMITS_CUSTOMER_ENABLED:false}" - configuration: "${TB_SERVER_REST_LIMITS_CUSTOMER_CONFIGURATION:50:1,1000:60}" server_side_rpc: # Minimum value of the server side RPC timeout. May override value provided in the REST API call. # Since 2.5 migration to queues, the RPC delay depends on the size of the pending messages in the queue, @@ -255,8 +236,6 @@ cassandra: # log one of cassandra queries with specified frequency (0 - logging is disabled) print_queries_freq: "${CASSANDRA_QUERY_PRINT_FREQ:0}" tenant_rate_limits: - enabled: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_ENABLED:false}" - configuration: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_CONFIGURATION:1000:1,30000:60}" print_tenant_names: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_PRINT_TENANT_NAMES:false}" # SQL configuration parameters diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 3d6669e870..0a1571a54e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -130,6 +130,11 @@ public class TenantProfile extends SearchTextBased implements H .map(profileConfiguration -> (DefaultTenantProfileConfiguration) profileConfiguration); } + @JsonIgnore + public DefaultTenantProfileConfiguration getDefaultProfileConfiguration() { + return getProfileConfiguration().orElse(null); + } + public TenantProfileData createDefaultTenantProfileData() { TenantProfileData tpd = new TenantProfileData(); tpd.setConfiguration(new DefaultTenantProfileConfiguration()); @@ -146,9 +151,4 @@ public class TenantProfile extends SearchTextBased implements H } } - @JsonIgnore - public DefaultTenantProfileConfiguration getDefaultTenantProfileConfiguration() { - return getProfileConfiguration().orElse(null); - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index e1b97d7f5a..2c3ea91873 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -46,6 +46,16 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private long maxTransportMessages; + private long maxTransportDataPoints; + private long maxREExecutions; + private long maxJSExecutions; + private long maxDPStorageDays; + private int maxRuleNodeExecutionsPerMessage; + private long maxEmails; + private long maxSms; + private long maxCreatedAlarms; + private String tenantServerRestLimitsConfiguration; private String customerServerRestLimitsConfiguration; @@ -62,16 +72,6 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String cassandraQueryTenantRateLimitsConfiguration; - private long maxTransportMessages; - private long maxTransportDataPoints; - private long maxREExecutions; - private long maxJSExecutions; - private long maxDPStorageDays; - private int maxRuleNodeExecutionsPerMessage; - private long maxEmails; - private long maxSms; - private long maxCreatedAlarms; - private int defaultStorageTtlDays; private int alarmsTtlDays; private int rpcTtlDays; diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index b478be0d7b..ef9d568896 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -116,7 +116,7 @@ public abstract class AbstractBufferedRateExecutor Date: Fri, 21 Jan 2022 15:31:53 +0200 Subject: [PATCH 019/262] Tenant profile rate limits configuration UI refactoring --- .../tenant-profile-data.component.html | 17 +++++-- .../profile/tenant-profile.component.html | 8 ++-- ...enant-profile-configuration.component.html | 48 +++++++------------ 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html index 4a979fb0cb..9af8d294de 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html @@ -16,8 +16,17 @@ -->
- - + + + +
tenant-profile.profile-configuration
+ + + + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html index 18dd97fac4..e5a3a54293 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html @@ -62,14 +62,14 @@
{{'tenant.isolated-tb-rule-engine-details' | translate}}
- - tenant-profile.description - - + + tenant-profile.description + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 9ea904d820..911b0b4bd2 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -213,12 +213,10 @@ - + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} - + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} @@ -262,32 +260,28 @@ tenant-profile.transport-tenant-msg-rate-limit - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} tenant-profile.transport-tenant-telemetry-msg-rate-limit - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} tenant-profile.transport-tenant-telemetry-data-points-rate-limit - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} tenant-profile.transport-device-msg-rate-limit - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} @@ -298,8 +292,7 @@ tenant-profile.transport-device-telemetry-data-points-rate-limit - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} @@ -308,8 +301,7 @@ - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} @@ -318,8 +310,7 @@ - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} @@ -340,8 +331,7 @@ tenant-profile.ws-limit-max-sessions-per-public-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} @@ -362,48 +352,42 @@ tenant-profile.ws-limit-max-subscriptions-per-tenant - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-customer - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-regular-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-public-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-updates-per-session - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} tenant-profile.cassandra-tenant-limits-configuration - + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} From 6267430dd989afaf3c2a34704456db37199e7c90 Mon Sep 17 00:00:00 2001 From: Kalutka Zhenya Date: Mon, 24 Jan 2022 11:34:37 +0200 Subject: [PATCH 020/262] Add ui-help --- .../profile/alarm/alarm-dynamic-value.component.html | 1 + .../profile/alarm/alarm-dynamic-value.component.ts | 7 +++++-- .../components/profile/alarm/alarm-schedule.component.html | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html index 50b2dee7a7..99eee091cb 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html @@ -48,6 +48,7 @@
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts index ad4b8cc02c..975331b7b4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts @@ -14,8 +14,9 @@ /// limitations under the License. /// -import { Component, forwardRef, OnInit } from '@angular/core'; +import { Component, forwardRef, Input, OnInit } from '@angular/core'; import { + AbstractControl, ControlValueAccessor, FormBuilder, FormGroup, @@ -43,12 +44,14 @@ export class AlarmDynamicValue implements ControlValueAccessor, OnInit{ public dynamicValueSourceTypeTranslations = dynamicValueSourceTypeTranslationMap; private propagateChange = (v: any) => { }; + @Input() helpId: string; + constructor(private fb: FormBuilder) { } ngOnInit(): void { this.dynamicValue = this.fb.group({ - sourceType: [null], + sourceType: [null, []], sourceAttribute: [null] }) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html index 67a3ac69c0..7cb5a3029e 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html @@ -34,7 +34,7 @@ formControlName="timezone">
- +
device-profile.schedule-days
@@ -74,7 +74,7 @@
- +
device-profile.schedule-days
From 5118b21f2662a8c8899ec9ec2d71f433e57805c8 Mon Sep 17 00:00:00 2001 From: Kalutka Zhenya Date: Fri, 28 Jan 2022 17:10:43 +0200 Subject: [PATCH 021/262] Added UI help for dynamic value in alarm schedule --- .../alarm_specific_schedule_format.md | 31 ++++++++ .../alarm_сustom_schedule_format.md | 79 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 ui-ngx/src/assets/help/en_US/device-profile/alarm_specific_schedule_format.md create mode 100644 ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md diff --git a/ui-ngx/src/assets/help/en_US/device-profile/alarm_specific_schedule_format.md b/ui-ngx/src/assets/help/en_US/device-profile/alarm_specific_schedule_format.md new file mode 100644 index 0000000000..cbf6b3a3ea --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/device-profile/alarm_specific_schedule_format.md @@ -0,0 +1,31 @@ +#### Specific schedule format + +An attribute with a dynamic value for a specific schedule format must have JSON in the following format: + +```javascript +{ + "daysOfWeek": [ + 2, + 4 + ], + "endsOn": 0, + "startsOn": 0, + "timezone": "Europe/Kiev" +} +``` + +
    +
  • +timezone: this value is used to designate the timezone you are using. +
    • +
  • +daysOfWeek: this value is used to designate the days in numerical representation (Monday - 1, Tuesday 2, etc.) on which the schedule will be active. +
    • +
  • +startsOn: this value is used to designate the timestamp in milliseconds, from which the schedule will be active for the designated days. +
    • +
  • +endsOn: this value is used to designate the timestamp in milliseconds until which the schedule will be active for the specified days. +
    • +
+When startsOn and endsOn equals 0 it's means that the schedule will be active the whole day. diff --git a/ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md b/ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md new file mode 100644 index 0000000000..7090ae68e1 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md @@ -0,0 +1,79 @@ +#### Custom schedule format + +An attribute with a dynamic value for a custom schedule format must have JSON in the following format: + +```javascript +{ + "timezone": "Europe/Kiev", + "items": [ + { + "dayOfWeek": 1, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 2, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 3, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 4, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 5, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 6, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + }, + { + "dayOfWeek": 7, + "enabled": true, + "endsOn": 0, + "startsOn": 0 + } + ] +} +``` + +
    +
  • +timezone: this value is used to designate the timezone you are using. +
    • +
  • +items: the array of values representing the days on which the schedule will be active. +
    • +
+ +One array item contains such fields: +
    +
  • +dayOfWeek: this value is used to designate the specified day in numerical representation (Monday - 1, Tuesday 2, etc.) on which the schedule will be active. +
    • +
  • +enabled: this boolean value, used to designate that the specified day in the schedule will be enabled. +
    • +
  • +startsOn: this value is used to designate the timestamp in milliseconds, from which the schedule will be active for the designated day. +
    • +
  • +endsOn: this value is used to designate the timestamp in milliseconds until which the schedule will be active for the specified day. +
    • +
+When startsOn and endsOn equals 0 it's means that the schedule will be active the whole day. From 6969882a699d3b61e0d91102b8965aed943bcbff Mon Sep 17 00:00:00 2001 From: Alexey Markevich Date: Fri, 28 Jan 2022 23:54:10 +0300 Subject: [PATCH 022/262] switch to PostgreSQL10Dialect --- application/src/main/resources/thingsboard.yml | 2 +- .../main/java/org/thingsboard/server/dao/util/PsqlDao.java | 2 +- .../org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java | 2 +- dao/src/test/resources/nosql-test.properties | 2 +- dao/src/test/resources/sql-test.properties | 4 ++-- docker/tb-node.hybrid.env | 2 +- docker/tb-node.postgres.env | 2 +- msa/tb/docker-cassandra/Dockerfile | 2 +- msa/tb/docker-postgres/Dockerfile | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 51494484db..d9fa8ef323 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -518,7 +518,7 @@ spring: open-in-view: "false" hibernate: ddl-auto: "none" - database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" + database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQL10Dialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index 81855d99cf..ba30f8975a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") +@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQL10Dialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java index ea44bd52e1..368f550c14 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java @@ -22,6 +22,6 @@ import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') " + - "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQL10Dialect'") public @interface PsqlTsLatestAnyDao { } diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index b81f5db8ac..24fa1b426f 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -11,7 +11,7 @@ spring.jpa.properties.hibernate.jdbc.log.warnings=false spring.jpa.show-sql=false spring.jpa.hibernate.ddl-auto=none -spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index d2add71eea..b85fb7abc1 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -12,7 +12,7 @@ spring.jpa.properties.hibernate.jdbc.log.warnings=false spring.jpa.show-sql=false spring.jpa.hibernate.ddl-auto=none -spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb @@ -32,7 +32,7 @@ service.type=monolith #spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true #spring.jpa.show-sql=false #spring.jpa.hibernate.ddl-auto=none -#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect # #spring.datasource.username=postgres #spring.datasource.password=postgres diff --git a/docker/tb-node.hybrid.env b/docker/tb-node.hybrid.env index 2f5dcee2e3..5f72c6e1a1 100644 --- a/docker/tb-node.hybrid.env +++ b/docker/tb-node.hybrid.env @@ -2,7 +2,7 @@ DATABASE_TS_TYPE=cassandra CASSANDRA_URL=cassandra:9042 -SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect +SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard SPRING_DATASOURCE_USERNAME=postgres diff --git a/docker/tb-node.postgres.env b/docker/tb-node.postgres.env index 63e8d11889..f4e11cd70a 100644 --- a/docker/tb-node.postgres.env +++ b/docker/tb-node.postgres.env @@ -1,7 +1,7 @@ # ThingsBoard server configuration for PostgreSQL database DATABASE_TS_TYPE=sql -SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect +SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard SPRING_DATASOURCE_USERNAME=postgres diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index 42dfebee45..144c359cc0 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -54,7 +54,7 @@ ENV DATABASE_TS_TYPE=cassandra ENV PGDATA=/data/db ENV CASSANDRA_DATA=/data/cassandra -ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect +ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect ENV SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver ENV SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/thingsboard ENV SPRING_DATASOURCE_USERNAME=${pkg.user} diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index 47e1a0dec6..6b6dbc0011 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -50,7 +50,7 @@ ENV DATABASE_TS_TYPE=sql ENV PGDATA=/data/db ENV PATH=$PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect +ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect ENV SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver ENV SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/thingsboard ENV SPRING_DATASOURCE_USERNAME=${pkg.user} From 15f28755549babb68bdfe341884b3680730a4677 Mon Sep 17 00:00:00 2001 From: "pinkevycdima@gmail.com" Date: Mon, 31 Jan 2022 14:57:56 +0200 Subject: [PATCH 023/262] UI: add copyDashboardId button in details-panel --- .../pages/dashboard/dashboard-form.component.html | 10 ++++++++++ .../home/pages/dashboard/dashboard-form.component.ts | 11 +++++++++++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ ui-ngx/src/assets/locale/locale.constant-ru_RU.json | 2 ++ ui-ngx/src/assets/locale/locale.constant-uk_UA.json | 2 ++ 5 files changed, 27 insertions(+) diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html index 0d8971395e..9bd5dd07ab 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-form.component.html @@ -66,6 +66,16 @@ {{'dashboard.delete' | translate }}
+
+ +
Date: Tue, 1 Feb 2022 18:09:08 +0200 Subject: [PATCH 024/262] Added disable mod --- .../alarm/alarm-dynamic-value.component.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts index 975331b7b4..910aa99962 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.ts @@ -16,7 +16,6 @@ import { Component, forwardRef, Input, OnInit } from '@angular/core'; import { - AbstractControl, ControlValueAccessor, FormBuilder, FormGroup, @@ -44,7 +43,11 @@ export class AlarmDynamicValue implements ControlValueAccessor, OnInit{ public dynamicValueSourceTypeTranslations = dynamicValueSourceTypeTranslationMap; private propagateChange = (v: any) => { }; - @Input() helpId: string; + @Input() + helpId: string; + + @Input() + disabled: boolean; constructor(private fb: FormBuilder) { } @@ -81,6 +84,15 @@ export class AlarmDynamicValue implements ControlValueAccessor, OnInit{ } } + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.dynamicValue.disable({emitEvent: false}); + } else { + this.dynamicValue.enable({emitEvent: false}); + } + } + private updateModel() { this.propagateChange(this.dynamicValue.value); } From ae85dae8bd8a7c89468155979f43056302655dea Mon Sep 17 00:00:00 2001 From: desoliture Date: Tue, 1 Feb 2022 18:23:06 +0200 Subject: [PATCH 025/262] fix processing isActive logic in custom schedules when endsOn equals zero --- .../thingsboard/rule/engine/profile/AlarmRuleState.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index f997efdeb3..ec7f38576e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -172,7 +172,13 @@ class AlarmRuleState { return false; } } - return isActive(eventTs, zoneId, zdt, schedule.getStartsOn(), schedule.getEndsOn()); + long endsOn = schedule.getEndsOn(); + if (endsOn == 0) { + // 24 hours in milliseconds + endsOn = 86400000; + } + + return isActive(eventTs, zoneId, zdt, schedule.getStartsOn(), endsOn); } private boolean isActiveCustom(CustomTimeSchedule schedule, long eventTs) { From 09d898c3a70a224b38c7d3e7ab89084426c17113 Mon Sep 17 00:00:00 2001 From: desoliture Date: Tue, 22 Feb 2022 18:00:12 +0200 Subject: [PATCH 026/262] update license headers --- .../components/profile/alarm/alarm-dynamic-value.component.html | 2 +- .../components/profile/alarm/alarm-dynamic-value.component.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html index 99eee091cb..6a64ba4800 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-dynamic-value.component.html @@ -1,6 +1,6 @@ 1.16.0 1.12 + 6.1.0.202203080745-r @@ -1875,6 +1876,11 @@ ${zeroturnaround.version} test + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + From b3dfed5badc997144dc69588aaef2e5ff44b0c79 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 12:00:02 +0300 Subject: [PATCH 083/262] API for listing entities at version, loading all entities at version, listing all available versions --- .../EntitiesVersionControlController.java | 64 +++++- .../DefaultEntitiesVersionControlService.java | 184 +++++++++++++----- .../vcs/EntitiesVersionControlService.java | 11 +- .../server/utils/git/Repository.java | 8 +- 4 files changed, 207 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 10928f1b51..798556998c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities/vc") @@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController { @PostMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, + @PathVariable("entityId") UUID id, @RequestParam String branch, @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); } + @PostMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); + } + + + @GetMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List listEntityVersions(@PathVariable EntityType entityType, @@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); } + @GetMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch) throws Exception { + return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); + } + + @GetMapping("/version") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listVersions(@RequestParam String branch) throws Exception { + return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/files/version/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listFilesAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); + } + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); } @PostMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); + onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + return result; + } + + @PostMapping("/entity/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> loadAllAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + SecurityUser user = getCurrentUser(); + List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); + resultList.forEach(result -> { + onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + }); + return resultList; } @GetMapping("/branches") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") public Set getAllowedBranches() throws ThingsboardException { return versionControlService.getAllowedBranches(getTenantId()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java index 0d78496f16..32da1309fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository; import org.thingsboard.server.utils.git.data.Commit; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -54,7 +57,9 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Service @@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private static final String SETTINGS_KEY = "vc"; private Repository repository; - private final ReentrantLock fetchLock = new ReentrantLock(); - private final Lock writeLock = new ReentrantLock(); + private final Lock fetchLock = new ReentrantLock(); + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); @AfterStartUp public void init() throws Exception { @@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - public void fetch() throws Exception { + private void fetch() throws Exception { if (repository == null) return; - - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); } @@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportOutboundRelations(false) .build(); List>> entityDataList = entitiesIds.stream() - .map(entityId -> { - return exportImportService.exportEntity(tenantId, entityId, exportSettings); - }) + .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) .collect(Collectors.toList()); - if (fetchLock.tryLock()) { - try { - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); - writeLock.lock(); + repositoryLock.writeLock().lock(); try { if (repository.listBranches().contains(branch)) { repository.checkout(branch); @@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); repository.push(); - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + return toVersion(commit); } finally { - writeLock.unlock(); + repositoryLock.writeLock().unlock(); } } @@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); } @Override public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); + return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); + } - return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + @Override + public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { + return listVersions(tenantId, branch, null, limit); + } + + private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, limit).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + + } finally { + repositoryLock.readLock().unlock(); + } } @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - checkRepository(); - // FIXME [viacheslav]: validate access + public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + return repository.listFilesAtCommit(versionId); + } finally { + repositoryLock.readLock().unlock(); + } + } - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return parseEntityData(entityDataJson); + } finally { + repositoryLock.readLock().unlock(); + } } + @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() .importInboundRelations(false) .importOutboundRelations(false) @@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + @Override + public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() + .map(entityDataFilePath -> { + String entityDataJson; + try { + entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return parseEntityData(entityDataJson); + }) + .collect(Collectors.toList()); + + return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + private void tryFetch() throws GitAPIException { + repositoryLock.readLock().lock(); + try { + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } finally { + repositoryLock.readLock().unlock(); + } + } private String getRelativePathForEntity(EntityId entityId) { @@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void checkBranch(TenantId tenantId, String branch) { + // TODO [viacheslav]: all branches are available by default? if (!getAllowedBranches(tenantId).contains(branch)) { throw new IllegalArgumentException("Tenant does not have access to this branch"); } @@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .orElse(Collections.emptySet()); } + private EntityVersion toVersion(Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } + + private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + + @Override public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - this.repository = initRepository(settings.getGitSettings()); + repositoryLock.writeLock().lock(); + try { + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); + } AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) .orElseGet(() -> { @@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } - private void checkRepository() { if (repository == null) { throw new IllegalStateException("Repository is not initialized"); @@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } public void resetRepository() throws Exception { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; + repositoryLock.writeLock().lock(); + try { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java index a9a3c0e3e8..e05ad17826 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -37,10 +37,17 @@ public interface EntitiesVersionControlService { List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; void saveSettings(EntitiesVersionControlSettings settings) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java index 8099be8259..23eb7d7db4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -113,13 +113,13 @@ public class Repository { } - public List listFilesAtCommit(Commit commit) throws IOException { - return listFilesAtCommit(commit, null); + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); } - public List listFilesAtCommit(Commit commit, String path) throws IOException { + public List listFilesAtCommit(String commitId, String path) throws IOException { List files = new ArrayList<>(); - RevCommit revCommit = resolveCommit(commit.getId()); + RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) { From 073875f406ed4437ec844a3a29a116effba5be44 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 28 Apr 2022 13:26:11 +0300 Subject: [PATCH 084/262] Entities VC improvements and refactoring --- .../EntitiesVersionControlController.java | 272 ++++++------ .../DefaultEntitiesVersionControlService.java | 395 ++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 64 +++ .../data/EntitiesVersionControlSettings.java} | 20 +- .../sync/{vcs => vc}/data/EntityVersion.java | 5 +- .../data/EntityVersionLoadResult.java} | 21 +- .../vc/data/EntityVersionLoadSettings.java} | 9 +- .../vc/data/EntityVersionSaveSettings.java} | 6 +- .../sync/vc/data/VersionedEntityInfo.java} | 11 +- .../DefaultEntitiesVersionControlService.java | 358 ---------------- .../vcs/EntitiesVersionControlService.java | 57 --- .../Repository.java => GitRepository.java} | 79 ++-- 12 files changed, 665 insertions(+), 632 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/GitSettings.java => vc/data/EntitiesVersionControlSettings.java} (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs => vc}/data/EntityVersion.java (90%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/EntitiesVersionControlSettings.java => vc/data/EntityVersionLoadResult.java} (57%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Commit.java => service/sync/vc/data/EntityVersionLoadSettings.java} (78%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Branch.java => service/sync/vc/data/EntityVersionSaveSettings.java} (83%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Diff.java => service/sync/vc/data/VersionedEntityInfo.java} (74%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/utils/{git/Repository.java => GitRepository.java} (82%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 798556998c..c39442341e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,159 +16,139 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @RestController @RequestMapping("/api/entities/vc") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { - private final DefaultEntitiesVersionControlService versionControlService; - - - - @PostMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID id, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); - return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); - } - - @PostMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); - return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); - } - - - - @GetMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityVersions(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityTypeVersions(@PathVariable EntityType entityType, - @RequestParam String branch) throws Exception { - return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listVersions(@RequestParam String branch) throws Exception { - return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); - } - - - - @GetMapping("/files/version/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listFilesAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); - } - - - - @GetMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); - } - - @PostMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); - onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - return result; - } - - @PostMapping("/entity/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> loadAllAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - SecurityUser user = getCurrentUser(); - List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); - resultList.forEach(result -> { - onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - }); - return resultList; - } - - - - @GetMapping("/branches") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public Set getAllowedBranches() throws ThingsboardException { - return versionControlService.getAllowedBranches(getTenantId()); - } - - - @PostMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { - versionControlService.saveSettings(settings); - } - - @GetMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public EntitiesVersionControlSettings getSettings() { - return versionControlService.getSettings(); - } - - - - @PostMapping("/repository/reset") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void resetLocalRepository() throws Exception { - versionControlService.resetRepository(); - } + private final EntitiesVersionControlService versionControlService; + + + // search request - export request with settings + +// +// @PostMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID id, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); +// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); +// } +// +// @PostMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, +// @RequestParam UUID[] ids, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// List entitiesIds = Arrays.stream(ids) +// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) +// .collect(Collectors.toList()); +// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); +// } +// +// +// +// @GetMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityVersions(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityTypeVersions(@PathVariable EntityType entityType, +// @RequestParam String branch) throws Exception { +// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listVersions(@RequestParam String branch) throws Exception { +// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); +// } +// +// +// +// @GetMapping("/files/version/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listFilesAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); +// } +// +// +// +// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); +// } +// +// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); +// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// return result; +// } +// +// @PostMapping("/entity/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List>> loadAllAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// SecurityUser user = getCurrentUser(); +// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); +// resultList.forEach(result -> { +// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// }); +// return resultList; +// } +// +// +// +// @GetMapping("/branches") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public Set getAllowedBranches() throws ThingsboardException { +// return versionControlService.getAllowedBranches(getTenantId()); +// } +// +// +// @PostMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { +// versionControlService.saveSettings(settings); +// } +// +// @GetMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public EntitiesVersionControlSettings getSettings() { +// return versionControlService.getSettings(); +// } +// +// +// +// @PostMapping("/repository/reset") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void resetLocalRepository() throws Exception { +// versionControlService.resetRepository(); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..a298d64ccc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,395 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.utils.GitRepository; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final EntitiesExportImportService exportImportService; + + private GitRepository repository; + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); + + private ScheduledExecutorService fetchExecutor; + private ScheduledFuture fetchTask; + + private final AdminSettingsService adminSettingsService; + private static final String SETTINGS_KEY = "vc"; + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + + + @AfterStartUp + public void init() { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null) { + try { + initRepository(settings); + } catch (Exception e) { + log.debug("Failed to init repository", e); + } + } + + int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); + fetchExecutor = Executors.newSingleThreadScheduledExecutor(); + fetchTask = scheduleFetch(fetchPeriod); + } + + + @Override + public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); + } + + @Override + public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + checkRepository(); + checkBranch(user.getTenantId(), branch); + + List> entityDataList = new ArrayList<>(); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(settings.isSaveRelations()) + .build(); + for (EntityId entityId : entitiesIds) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); + } + + GitRepository.Commit commit = repository.commit(versionName, "."); + repository.push(); + return toVersion(commit); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @Override + public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch) throws Exception { + return listVersions(tenantId, branch, null); + } + + private List listVersions(TenantId tenantId, String branch, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + } + + @Override + public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + checkVersion(tenantId, branch, versionId, path); + + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + info.setEntityName(entityData.getEntity().getName()); + return info; + }) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); + } + + @Override + public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); + } + + @Override + public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, null, settings); + } + + private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { + List> entityDataList = new ArrayList<>(); + repositoryLock.readLock().lock(); + try { + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { + EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + entityDataList.add(entityData); + } + } finally { + repositoryLock.readLock().unlock(); + } + + EntityImportSettings importSettings = EntityImportSettings.builder() + .updateRelations(settings.isLoadRelations()) + .findExistingByName(settings.isFindExistingEntityByName()) + .build(); + List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); + + return importResults.stream() + .map(importResult -> EntityVersionLoadResult.builder() + .previousEntityVersion(importResult.getOldEntity()) + .newEntityVersion(importResult.getSavedEntity()) + .entityType(importResult.getEntityType()) + .build()) + .collect(Collectors.toList()); + } + + @SneakyThrows + private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { + repositoryLock.readLock().lock(); + try { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + private void fetch() throws GitAPIException { + repositoryLock.writeLock().lock(); + try { + repository.fetch(); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private ScheduledFuture scheduleFetch(int fetchPeriod) { + return fetchExecutor.scheduleWithFixedDelay(() -> { + if (repository == null) return; + try { + fetch(); + } catch (Exception e) { + log.error("Failed to fetch remote repository", e); + } + }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + } + + + private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Version not found"); + } + } + + @Override + public List listAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptyList()); + } + + private void checkBranch(TenantId tenantId, String branch) { + if (!listAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to the branch"); + } + } + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private void initRepository(EntitiesVersionControlSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { + this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(Path.of(settings.getRepositoryDirectory())); + this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), + settings.getUsername(), settings.getPassword()); + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private void clearRepository() throws IOException { + repositoryLock.writeLock().lock(); + try { + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repository = null; + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @SneakyThrows + @Override + public void saveSettings(EntitiesVersionControlSettings settings) { + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .orElseGet(() -> { + AdminSettings newAdminSettings = new AdminSettings(); + newAdminSettings.setKey(SETTINGS_KEY); + return newAdminSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + + repositoryLock.writeLock().lock(); + try { + clearRepository(); + initRepository(settings); + } finally { + repositoryLock.writeLock().unlock(); + } + + if (settings.getFetchPeriod() != 0) { + fetchTask.cancel(true); + fetchTask = scheduleFetch(settings.getFetchPeriod()); + } + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..435c661b02 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + + List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + + List listVersions(TenantId tenantId, String branch) throws Exception; + + + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + + List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + + List listAllowedBranches(TenantId tenantId); + + + void saveSettings(EntitiesVersionControlSettings settings); + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 0d97bed2d1..06e0a9c7aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -13,20 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; +import java.util.UUID; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class GitSettings { - private String repositoryUri; +public class EntitiesVersionControlSettings { private String repositoryDirectory; + private String repositoryUri; private String username; private String password; + + private int fetchPeriod; + + private Map> tenantsAllowedBranches; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java index 42a2e2f555..3547f89e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@NoArgsConstructor @AllArgsConstructor +@NoArgsConstructor public class EntityVersion { private String id; private String name; - private String authorName; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java index 6a46606845..4cf0ee2b0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java @@ -13,16 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; - -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; @Data -public class EntitiesVersionControlSettings { - private Map> allowedBranches; - private GitSettings gitSettings; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionLoadResult { + private ExportableEntity newEntityVersion; + private ExportableEntity previousEntityVersion; + private EntityType entityType; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java index 7567dba1c5..842f6ddd4e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Commit { - private final String id; - private final String message; - private final String authorName; +public class EntityVersionLoadSettings { + private boolean loadRelations; + private boolean findExistingEntityByName; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java index a458f9f5aa..1fc36463ba 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Branch { - private final String shortName; +public class EntityVersionSaveSettings { + private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 7572a003d2..97bb6ada12 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; @Data -public class Diff { - private final String type; - private final String oldPath; - private final String newPath; +public class VersionedEntityInfo { + private EntityId externalId; + private String entityName; + // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java deleted file mode 100644 index 32da1309fa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; -import org.thingsboard.server.service.sync.vcs.data.GitSettings; -import org.thingsboard.server.utils.git.Repository; -import org.thingsboard.server.utils.git.data.Commit; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - // TODO [viacheslav]: start up only on one of the cores - - private final TenantService tenantService; - private final EntitiesExportImportService exportImportService; - private final AdminSettingsService adminSettingsService; - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private static final String SETTINGS_KEY = "vc"; - - private Repository repository; - private final Lock fetchLock = new ReentrantLock(); - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - @AfterStartUp - public void init() throws Exception { - try { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null && settings.getGitSettings() != null) { - this.repository = initRepository(settings.getGitSettings()); - } - } catch (Exception e) { - log.error("Failed to initialize entities version control service", e); - } - } - - - @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - private void fetch() throws Exception { - if (repository == null) return; - tryFetch(); - } - - - @Override - public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { - return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); - } - - @Override - public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(false) - .exportOutboundRelations(false) - .build(); - List>> entityDataList = entitiesIds.stream() - .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) - .collect(Collectors.toList()); - - tryFetch(); - - repositoryLock.writeLock().lock(); - try { - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); - } - - for (EntityExportData> entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), - entityDataJson, StandardCharsets.UTF_8); - } - - Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); - repository.push(); - return toVersion(commit); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - - - @Override - public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); - } - - @Override - public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); - } - - @Override - public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, null, limit); - } - - private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { - repositoryLock.readLock().lock(); - try { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, path, limit).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - return repository.listFilesAtCommit(versionId); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return parseEntityData(entityDataJson); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); - return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } - - @Override - public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() - .map(entityDataFilePath -> { - String entityDataJson; - try { - entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return parseEntityData(entityDataJson); - }) - .collect(Collectors.toList()); - - return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } finally { - repositoryLock.readLock().unlock(); - } - } - - private void tryFetch() throws GitAPIException { - repositoryLock.readLock().lock(); - try { - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } - } finally { - repositoryLock.readLock().unlock(); - } - } - - - private String getRelativePathForEntity(EntityId entityId) { - return getRelativePathForEntityType(entityId.getEntityType()) - + "/" + entityId.getId() + ".json"; - } - - private String getRelativePathForEntityType(EntityType entityType) { - return entityType.name().toLowerCase(); - } - - - private void checkBranch(TenantId tenantId, String branch) { - // TODO [viacheslav]: all branches are available by default? - if (!getAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to this branch"); - } - } - - public Set getAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptySet()); - } - - private EntityVersion toVersion(Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); - } - - private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); - } - - - - @Override - public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .orElseGet(() -> { - AdminSettings newSettings = new AdminSettings(); - newSettings.setKey(SETTINGS_KEY); - return newSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - } - - @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) - .orElse(null); - } - - - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } - } - - private static Repository initRepository(GitSettings gitSettings) throws Exception { - if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { - return Repository.open(gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } else { - Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); - return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } - } - - public void resetRepository() throws Exception { - repositoryLock.writeLock().lock(); - try { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; - } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java deleted file mode 100644 index e05ad17826..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.List; - -public interface EntitiesVersionControlService { - - EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; - - EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; - - - List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; - - List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; - - List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - - - List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - void saveSettings(EntitiesVersionControlSettings settings) throws Exception; - - EntitiesVersionControlSettings getSettings(); - -} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/utils/git/Repository.java rename to application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 23eb7d7db4..205bbca50e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git; +package org.thingsboard.server.utils; import com.google.common.collect.Streams; +import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.Git; @@ -32,19 +33,21 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.utils.git.data.Commit; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class Repository { +public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; @@ -52,13 +55,13 @@ public class Repository { @Getter private final String directory; - private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; this.directory = directory; } - public static Repository clone(String uri, String directory, + public static GitRepository clone(String uri, String directory, String username, String password) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() @@ -67,12 +70,12 @@ public class Repository { .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new Repository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory); } - public static Repository open(String directory, String username, String password) throws IOException { + public static GitRepository open(String directory, String username, String password) throws IOException { Git git = Git.open(new java.io.File(directory)); - return new Repository(git, newCredentialsProvider(username, password), directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory); } @@ -81,6 +84,20 @@ public class Repository { .setRemoveDeletedRefs(true)); } + public void checkout(String branch) throws GitAPIException { + execute(git.checkout() + .setName(branch)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + public List listBranches() throws GitAPIException { return execute(git.branchList() @@ -92,12 +109,12 @@ public class Repository { } - public List listCommits(String branchName, int limit) throws IOException, GitAPIException { - return listCommits(branchName, null, limit); + public List listCommits(String branch, int limit) throws IOException, GitAPIException { + return listCommits(branch, null, limit); } - public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); + public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } @@ -150,20 +167,6 @@ public class Repository { } - public void checkout(String branchName) throws GitAPIException { - execute(git.checkout() - .setName(branchName)); - } - - public void merge(String branchName) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); - if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); - } - execute(git.merge() - .include(branchId)); - } - public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) @@ -177,15 +180,11 @@ public class Repository { execute(git.clean()); } - public void clean() throws GitAPIException { - execute(git.clean().setCleanDirectories(true)); - } - public Commit commit(String message, String filePattern, String author) throws GitAPIException { - execute(git.add().addFilepattern(filePattern)); + public Commit commit(String message, String filesPattern) throws GitAPIException { + execute(git.add().addFilepattern(filesPattern)); RevCommit revCommit = execute(git.commit() - .setMessage(message) - .setAuthor(author, author)); + .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); } @@ -239,12 +238,8 @@ public class Repository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand) { + if (command instanceof TransportCommand && credentialsProvider != null) { ((TransportCommand) command).setCredentialsProvider(credentialsProvider); -// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); -// transportCommand.setTransportConfigCallback(transport -> { -// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); -// }); } return command.call(); } @@ -253,4 +248,12 @@ public class Repository { return new UsernamePasswordCredentialsProvider(username, password); } + + @Data + public static class Commit { + private final String id; + private final String message; + private final String authorName; + } + } From 074ea8af6cdd9d723f57dd55615cf602d16738a8 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 16:09:00 +0300 Subject: [PATCH 085/262] VC refactoring --- .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesExportImportController.java | 206 -------- .../EntitiesVersionControlController.java | 293 ++++++----- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 30 +- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 56 ++- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 12 +- .../exporting/data/DeviceExportData.java | 2 +- .../exporting/data/EntityExportData.java | 2 +- .../exporting/data}/EntityExportSettings.java | 2 +- .../exporting/data/RuleChainExportData.java | 2 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 9 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/data/EntityImportResult.java | 2 +- .../importing/data/EntityImportSettings.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../importing/data/request/ImportRequest.java | 32 -- .../DefaultEntitiesVersionControlService.java | 463 +++++++++++------- .../vc/EntitiesVersionControlService.java | 27 +- .../data/EntitiesVersionControlSettings.java | 10 +- .../sync/vc/data/VersionCreationResult.java | 11 + ...LoadResult.java => VersionLoadResult.java} | 8 +- ...iesByCustomFilterVersionCreateConfig.java} | 14 +- ...tiesByCustomQueryVersionCreateConfig.java} | 12 +- .../EntityListVersionCreateConfig.java} | 10 +- .../EntityTypeVersionCreateConfig.java} | 14 +- .../SingleEntityVersionCreateConfig.java} | 10 +- .../request/create/VersionCreateConfig.java} | 18 +- .../create/VersionCreateConfigType.java} | 4 +- .../create/VersionCreateRequest.java} | 14 +- .../load/EntityTypeVersionLoadConfig.java | 12 + .../load/EntityTypeVersionLoadRequest.java | 20 + .../request/load/EntityVersionLoadConfig.java | 11 + .../load/SingleEntityVersionLoadRequest.java | 20 + .../load/VersionLoadRequest.java} | 11 +- .../request/load/VersionLoadRequestType.java | 6 + .../server/utils/GitRepository.java | 39 +- ...aseEntitiesExportImportControllerTest.java | 20 +- ...EntitiesExportImportControllerSqlTest.java | 53 +- .../org/thingsboard/server/dao/DaoUtil.java | 20 +- .../server/dao/ExportableEntityDao.java | 4 + .../server/dao/sql/asset/JpaAssetDao.java | 5 + .../dao/sql/customer/JpaCustomerDao.java | 5 + .../sql/dashboard/DashboardRepository.java | 4 + .../dao/sql/dashboard/JpaDashboardDao.java | 7 + .../server/dao/sql/device/JpaDeviceDao.java | 5 + .../dao/sql/device/JpaDeviceProfileDao.java | 5 + .../server/dao/sql/rule/JpaRuleChainDao.java | 5 + 68 files changed, 860 insertions(+), 777 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/DefaultEntitiesExportImportService.java (84%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/EntitiesExportImportService.java (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/DefaultExportableEntitiesService.java (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/ExportableEntitiesService.java (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/DeviceExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/EntityExportData.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request => exportimport/exporting/data}/EntityExportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/RuleChainExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/AbstractBulkImportService.java (97%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportResult.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/RuleChainImportService.java (95%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadResult.java => VersionLoadResult.java} (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityFilterExportRequest.java => vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java} (69%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityQueryExportRequest.java => vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java} (72%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityListExportRequest.java => vc/data/request/create/EntityListVersionCreateConfig.java} (73%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityTypeExportRequest.java => vc/data/request/create/EntityTypeVersionCreateConfig.java} (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/SingleEntityExportRequest.java => vc/data/request/create/SingleEntityVersionCreateConfig.java} (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequest.java => vc/data/request/create/VersionCreateConfig.java} (58%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequestType.java => vc/data/request/create/VersionCreateConfigType.java} (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadSettings.java => request/create/VersionCreateRequest.java} (73%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionSaveSettings.java => request/load/VersionLoadRequest.java} (74%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index e48f2dcf23..66b570cf1d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -54,8 +54,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 03f0996ea9..4b6eca7879 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,8 +74,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8f45427603..948a380fcd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -55,8 +55,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java deleted file mode 100644 index c91fd0c292..0000000000 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import io.swagger.annotations.ApiOperation; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; - -@RestController -@RequestMapping("/api/entities") -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class EntitiesExportImportController extends BaseController { - - private final EntitiesExportImportService exportImportService; - private final ExportableEntitiesService exportableEntitiesService; - - - @ApiOperation(value = "Export entities by request", notes = "" + - "Takes export request and returns list of export data for each entity found by request. " + - "Supported entity types for export, hence for import, are **DEVICE**, **DEVICE_PROFILE**, **ASSET**, " + - "**CUSTOMER**, **RULE_CHAIN** and **DASHBOARD**." + NEW_LINE + - "For each type of export request, you can set some export settings: \n" + - "- **exportRelations** - whether to export inbound and outbound relations for an entity " + - "(only relations of type group COMMON can be exported)" + NEW_LINE + - "Supported export requests:\n" + - "- **SINGLE_ENTITY**:" + NEW_LINE + - " To export a single entity by id. Example:" + NEW_LINE + - "```\n{\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **ENTITY_LIST**:" + NEW_LINE + - " To export a list of entities by their ids. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"ASSET\",\n \"id\": \"2f0a3bd0-989d-11ec-93b5-6de6c2b68078\"\n }\n ],\n \"exportSettings\": {\n \"exportRelations\": true\n }\n}\n```" + NEW_LINE + - "- **ENTITY_TYPE**:" + NEW_LINE + - " To export entities of specified entity type. You need to specify page size, " + - "and may specify page index and customer id (to limit the list of entities to the ones owned by a customer). " + - "Entities are ordered by created time descendingly. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_FILTER**:" + NEW_LINE + - " To export entities by custom entity filter. The order used is the same as for ENTITY_TYPE export request. Example:" + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"Thermostats\",\n \"deviceNameFilter\": \"\"\n },\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_QUERY**:" + NEW_LINE + - " To export entities by custom entity query. Example: " + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_QUERY\",\n \"query\": {\n \"entityFilter\": {\n \"type\": \"entityType\",\n \"entityType\": \"DEVICE\"\n },\n \"pageLink\": {\n \"page\": 0,\n \"pageSize\": 200,\n \"textSearch\": \"THB_\",\n \"sortOrder\": {\n \"key\": {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n },\n \"direction\": \"DESC\"\n }\n },\n \"entityFields\": [\n {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n }\n ],\n \"latestValues\": [\n {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n }\n ],\n \"keyFilters\": [\n {\n \"key\": {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n },\n \"valueType\": \"NUMERIC\",\n \"predicate\": {\n \"type\": \"NUMERIC\",\n \"operation\": \"GREATER\",\n \"value\": {\n \"defaultValue\": 0\n }\n }\n }\n ]\n },\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "Mostly, export data of an entity contains the whole entity itself and its relations " + - "(if option to export relations was enabled):" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"ASSET\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1648204424029,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": { ... },\n \"customerId\": { ... },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n ...\n },\n \"relations\": [\n {\n \"from\": {\n \"entityType\": \"ASSET\",\n \"id\": ...\n },\n \"to\": {\n \"entityType\": \"DEVICE\",\n \"id\": ...\n },\n \"type\": \"Contains\",\n \"typeGroup\": \"COMMON\",\n \"additionalInfo\": {\n \"a\": \"b\"\n }\n }\n ]\n }\n]\n```" + NEW_LINE + - "For devices, export data will additionally contain device's credentials; for rule chains - its metadata:" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"DEVICE\",\n \"entity\": { ... },\n \"credentials\": {\n \"id\": { ... },\n \"createdTime\": 1648829321209,\n \"deviceId\": { ... },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"5cZEDo45KGW7JgVNv4Ko\",\n \"credentialsValue\": null\n }\n }\n]\n```" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"RULE_CHAIN\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1646056614257,\n \"additionalInfo\": null,\n \"tenantId\": { ... },\n \"name\": \"Rule Chain 2\",\n \"type\": \"CORE\",\n \"firstRuleNodeId\": { ... },\n \"root\": false,\n ...\n },\n \"metaData\": {\n \"ruleChainId\": { ... },\n \"firstNodeIndex\": 7,\n \"nodes\": [ ... ],\n \"connections\": [ ... ],\n \"ruleChainConnections\": null\n }\n }\n]\n```" + NEW_LINE + - "Returned export data is to be used later for import request." + NEW_LINE + - "If any entity found by request is of unsupported type - an error will be returned.\n" + - "Also, if a user does not have a READ permission for an entity (or, if relations are exported, for a bounded entity), " + - "access will be denied." + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping("/export") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - return exportEntitiesByRequest(user, exportRequest); - } catch (Exception e) { - log.warn("Failed to export entities for request {}", exportRequest, e); - throw handleException(e); - } - } - - @ApiOperation(value = "Export entities by multiple requests", notes = "" + - "The API behaviour is the same as for exporting entities by single request, " + - "except that this method takes an array of export requests as a request body." + NEW_LINE + - "Example:" + NEW_LINE + - "```\n[\n {\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"5f9eda10-b442-11ec-bbf5-adec34031568\"\n }\n },\n {\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"thermostat\",\n \"deviceNameFilter\": \"\"\n },\n \"pageSize\": 1000\n },\n {\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"pageSize\": 1000,\n \"exportSettings\": {\n \"exportRelations\": true\n }\n },\n {\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"2ef13590-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"e7311ec0-b442-11ec-bbf5-adec34031568\"\n }\n ]\n }\n]\n```" + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping(value = "/export", params = {"multiple"}) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - List> result = new ArrayList<>(); - for (ExportRequest exportRequest : exportRequests) { - List> exportDataList = exportEntitiesByRequest(user, exportRequest); - result.addAll(exportDataList); - } - return result; - } catch (Exception e) { - log.warn("Failed to export entities for requests {}", exportRequests, e); - throw handleException(e); - } - } - - private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { - List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); - - EntityExportSettings exportSettings = exportRequest.getExportSettings(); - if (exportSettings == null) { - exportSettings = EntityExportSettings.builder() - .exportRelations(false) - .build(); - } - - List> exportDataList = new ArrayList<>(); - for (EntityId entityId : entities) { - EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportSettings); - exportDataList.add(exportData); - } - return exportDataList; - } - - - @ApiOperation(value = "Import entities by request", notes = "" + - "Takes import request and returns the list of import results. " + - "Import request must contain the list of export data and might contain import settings. " + NEW_LINE + - "The method creates an entity if it is new in the scope of a tenant, or otherwise updates an existing one. " + - "On entity import request, we first try to find an entity within a tenant that has externalId equal " + - "to the id in the export data. If the platform fails to do that, we then search for an entity with " + - "regular (internal) id like the one in the export data (this is useful in case we are exporting and " + - "importing entities within the same tenant). Then, if we still haven't found any entity, if findExistingByName " + - "option of the EntityImportSettings is enabled, we will search for the one by its name (this is also useful " + - "for avoiding conflicts with default device profile or Root Rule Chain when importing all entities from another " + - "tenant). After, if the exported entity is new for this tenant, we simply save it with external id " + - "from the export data, and also create relations if any (if updateRelations option is enabled). " + - "Otherwise, we will reset all fields of the existing entity to the ones from the export data and save it, " + - "and also will update the list of relations (remove the ones that aren't present in the export data, " + - "and update or create others), if updateRelations option is enabled." + NEW_LINE + - "If an entity contains references to some other entities, like device references certain device profile, " + - "we will find this other entity within the tenant by this principle: look for an entity with " + - "such external id, or otherwise, internal id. This requires referenced entities to be imported " + - "before the referencing entity (if we are importing to another tenant). So, when receiving the list " + - "of entities' export data for import, we first try to fix the data order to import 'standalone' entities first." + NEW_LINE + - "As for relations importing, they are processed after all entities in the import batch are already saved, " + - "and the internal id of a bounded entity is found with the regular principle." + NEW_LINE + - "Import of all entities and their relations from the import request is processed in the single transaction, " + - "and so everything will be rolled back if the platform fails to e.g. find internal entity by external id. \n" + - "Example of import request:\n" + - "```\n{\n \"importSettings\": {\n \"findExistingByName\": false,\n \"updateRelations\": false\n },\n \"exportDataList\": [\n {\n \"entityType\": \"DEVICE_PROFILE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n \"createdTime\": 1649096026765,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"name\": \"Profile 1\",\n ...\n }\n },\n {\n \"entityType\": \"DEVICE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154276962,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"customerId\": {\n \"entityType\": \"CUSTOMER\",\n \"id\": \"13814000-1dd2-11b2-8080-808080808080\"\n },\n \"name\": \"Device 1\",\n \"type\": \"Profile 1\",\n \"label\": \"v1.0\",\n \"deviceProfileId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n ...\n },\n \"credentials\": {\n \"id\": {\n \"id\": \"981e0360-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154277014,\n \"deviceId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"sGExNdnl71uKmkNvtNdp\",\n \"credentialsValue\": null\n }\n }\n ]\n}\n```" + NEW_LINE + - "The response contains a list of EntityImportResult which has values of savedEntity and oldEntity:\n" + - "```\n[\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v2.0\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"6b03ab20-989e-11ec-b446-89df822d7fa2\"\n }\n },\n \"oldEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v1.0\",\n ...\n \"externalId\": null\n },\n \"entityType\": \"ASSET\"\n },\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"387213f0-b4ea-11ec-abfc-6dcc6508d0b5\"\n },\n \"createdTime\": 1649167860399,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 2\",\n \"type\": \"B\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"0b9ea0d0-ac27-11ec-a2a6-89d15eae3b21\"\n }\n },\n \"oldEntity\": null,\n \"entityType\": \"ASSET\"\n }\n]\n```") - @PostMapping("/import") - public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - EntityImportSettings importSettings = importRequest.getImportSettings(); - if (importSettings == null) { - importSettings = EntityImportSettings.builder() - .findExistingByName(false) - .updateRelations(false) - .build(); - } - - List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importSettings); - - importResults.stream() - .map(EntityImportResult::getSendEventsCallback) - .filter(Objects::nonNull) - .forEach(sendEventsCallback -> { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send event for entity", e); - } - }); - - return importResults; - } catch (Exception e) { - log.warn("Failed to import entities for request {}", importRequest, e); - throw handleException(e); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index c39442341e..0625dbeb38 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,140 +15,189 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; @RestController @RequestMapping("/api/entities/vc") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; - // search request - export request with settings - -// -// @PostMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID id, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); -// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); -// } -// -// @PostMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, -// @RequestParam UUID[] ids, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// List entitiesIds = Arrays.stream(ids) -// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) -// .collect(Collectors.toList()); -// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); -// } -// -// -// -// @GetMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityVersions(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityTypeVersions(@PathVariable EntityType entityType, -// @RequestParam String branch) throws Exception { -// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listVersions(@RequestParam String branch) throws Exception { -// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); -// } -// -// -// -// @GetMapping("/files/version/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listFilesAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); -// } -// -// -// -// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); -// } -// -// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); -// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// return result; -// } -// -// @PostMapping("/entity/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List>> loadAllAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// SecurityUser user = getCurrentUser(); -// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); -// resultList.forEach(result -> { -// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// }); -// return resultList; -// } -// -// -// -// @GetMapping("/branches") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public Set getAllowedBranches() throws ThingsboardException { -// return versionControlService.getAllowedBranches(getTenantId()); -// } -// -// -// @PostMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { -// versionControlService.saveSettings(settings); -// } -// -// @GetMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public EntitiesVersionControlSettings getSettings() { -// return versionControlService.getSettings(); -// } -// -// -// -// @PostMapping("/repository/reset") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void resetLocalRepository() throws Exception { -// versionControlService.resetRepository(); -// } + @PostMapping("/version") + public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + return versionControlService.saveEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") + public List listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { + try { + EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}/{entityType}") + public List listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { + try { + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}") + public List listVersions(@PathVariable String branch) throws ThingsboardException { + try { + return versionControlService.listVersions(getTenantId(), branch); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/entity/{branch}/{entityType}/{versionId}") + public List listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/entity/{branch}/{versionId}") + public List listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + + @PostMapping("/entity") + public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + String versionId = request.getVersionId(); + if (versionId == null) { + List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); + if (versions.size() > 0) { + versionId = versions.get(0).getId(); + } else { + throw new IllegalArgumentException("No versions available in branch"); + } + } + + return versionControlService.loadEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "") + @GetMapping("/branches") + public List listBranches() throws ThingsboardException { + try { + List remoteBranches = versionControlService.listBranches(getTenantId()); + List infos = new ArrayList<>(); + + String defaultBranch = getSettings().getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } + + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @GetMapping("/settings") + public EntitiesVersionControlSettings getSettings() throws ThingsboardException { + try { + return versionControlService.getSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @PostMapping("/settings") + public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + versionControlService.saveSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + + @Data + public static class BranchInfo { + private final String name; + private final boolean isDefault; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index cfefcafbf4..179c9c697d 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 3aac9c3641..3e63652b97 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index b09ac532df..75f7133721 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 55384b21db..608a256514 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,14 +27,14 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; import java.util.ArrayList; @@ -74,6 +74,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { fixOrder(exportDataList); + List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { @@ -88,6 +89,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS saveReferencesCallback.run(); } + importResults.stream() + .map(EntityImportResult::getSendEventsCallback) + .filter(Objects::nonNull) + .forEach(sendEventsCallback -> { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + return importResults; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index eaaf01ba2c..5c29feb57c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index ce6924de93..4d772ad935 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -37,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.entity.EntityService; @@ -46,12 +47,12 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -129,6 +130,16 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi return entity; } + @Override + public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + return exportableEntityDao.findByTenantId(tenantId.getId(), pageLink); + } + return new PageData<>(); + } + private boolean belongsToTenant(HasId entity, TenantId tenantId) { return tenantId.equals(((HasTenantId) entity).getTenantId()); } @@ -136,31 +147,31 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi @Transactional(readOnly = true, timeout = 40) @Override - public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { + public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { switch (request.getType()) { case SINGLE_ENTITY: { - return List.of(((SingleEntityExportRequest) request).getEntityId()); + return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); } case ENTITY_LIST: { - return ((EntityListExportRequest) request).getEntitiesIds(); + return ((EntityListVersionCreateConfig) request).getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); entityTypeFilter.setEntityType(exportRequest.getEntityType()); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_FILTER: { - CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; + EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; EntityFilter filter = exportRequest.getFilter(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_QUERY: { - CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; + EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; EntityDataQuery query = exportRequest.getQuery(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); @@ -194,6 +205,17 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } } + @Override + public void deleteByTenantIdAndId(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + + dao.removeById(tenantId, id.getId()); + } + @Override public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java index 73f946e722..5532f30125 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 9021a03e85..043ec91e20 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,9 +21,11 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import java.util.List; @@ -35,7 +37,11 @@ public interface ExportableEntitiesService { , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); - List findEntitiesForRequest(TenantId tenantId, ExportRequest request); + , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + + List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: + + void deleteByTenantIdAndId(TenantId tenantId, I id); void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java index 56633f9c91..4be338d7cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java index 728d8212c4..81e3b344d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java index 5454105ee5..0fb318b5fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java index 2e05e2f1f5..e1d870473f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java index 7b5f758f93..46428a763f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java index a051f0e59f..69af613480 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java index 21df8beda2..93f9c51655 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java index e814cd6ce7..f40ee6884a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java index 6c82530273..be6f62efe4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.exportimport.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java index 06cd7a93be..b2bfafc695 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -166,7 +165,7 @@ public abstract class AbstractBulkImportService data) { + private void saveKvs(SecurityUser user, E entity, Map data) { Arrays.stream(BulkImportColumnType.values()) .filter(BulkImportColumnType::isKv) .map(kvType -> { @@ -250,7 +249,7 @@ public abstract class AbstractBulkImportService columnsMappings = request.getMapping().getColumns(); + List columnsMappings = request.getMapping().getColumns(); return records.stream() .map(record -> { EntityData entityData = new EntityData(); @@ -281,7 +280,7 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); - private final Map kvs = new LinkedHashMap<>(); + private final Map kvs = new LinkedHashMap<>(); private int lineNumber; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java index beafe5e95f..9c0e6b6491 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java index f600289ab6..6347910cab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java index da12a6baeb..550d4a72f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java index 68e7a49b18..a39dcc7481 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java index 62c574d5f0..242856bfcb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java index a5de7d8744..075a6896ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java index ae54a988f3..80a942c73f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java index 28095a7b85..7ecfa7d3ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java index 1a79fe666b..b69c5a0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java index 77f32bf94c..b85e403f87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java index 091b6eb856..112152365b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java index 5bc45dc567..f97950aaf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java index 09a13d5f14..993f9fa27a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java deleted file mode 100644 index 7943e1a404..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.importing.data.request; - -import lombok.Data; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; - -import java.util.List; - -@Data -public class ImportRequest { - - private List> exportDataList; - private EntityImportSettings importSettings; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index a298d64ccc..919960ce62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,35 +18,62 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import org.thingsboard.server.utils.GitRepository; import java.io.File; @@ -56,16 +83,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -73,75 +103,152 @@ import java.util.stream.Collectors; public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { private final EntitiesExportImportService exportImportService; + private final ExportableEntitiesService exportableEntitiesService; + private final AttributesService attributesService; + private final EntityService entityService; + private final TenantDao tenantDao; - private GitRepository repository; - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - private ScheduledExecutorService fetchExecutor; - private ScheduledFuture fetchTask; + // TODO [viacheslav]: concurrency + private final Map repositories = new ConcurrentHashMap<>(); + @Value("${java.io.tmpdir}") + private String repositoriesFolder; - private final AdminSettingsService adminSettingsService; private static final String SETTINGS_KEY = "vc"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null) { - try { - initRepository(settings); - } catch (Exception e) { - log.debug("Failed to init repository", e); + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } } - } - - int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); - fetchExecutor = Executors.newSingleThreadScheduledExecutor(); - fetchTask = scheduleFetch(fetchPeriod); + }); + Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, 5, 5, TimeUnit.SECONDS); } @Override - public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); - } + public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); + repository.getLock().writeLock().lock(); - @Override - public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - repositoryLock.writeLock().lock(); try { - checkRepository(); - checkBranch(user.getTenantId(), branch); - - List> entityDataList = new ArrayList<>(); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(settings.isSaveRelations()) - .build(); - for (EntityId entityId : entitiesIds) { - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // FIXME [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); + for (VersionCreateRequest.Config config : request.getConfigs()) { + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build(); + + List> entityDataList = new ArrayList<>(); + for (EntityId entityId : findEntities()) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + if (config.isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() + .map(EntityExportData::getEntityType) + .distinct() + .forEach(entityType -> { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + } } + // TODO [viacheslav]: find with pagination - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); - } + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - GitRepository.Commit commit = repository.commit(versionName, "."); + GitRepository.Commit commit = repository.commit(request.getVersionName()); repository.push(); - return toVersion(commit); + + result.setVersion(toVersion(commit)); + return result; } finally { - repositoryLock.writeLock().unlock(); + repository.getLock().writeLock().unlock(); + } + } + + private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { + switch (entityFilter.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + return List.of(filter.getEntityId()); + } + case ENTITY_LIST: { + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + return filter.getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); + entityTypeFilter.setEntityType(filter.getEntityType()); + + newFilter.setFilter(entityTypeFilter); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_FILTER: { + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); + + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + newFilter.setQuery(query); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_QUERY: { + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); + return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() + .stream().map(EntityData::getEntityId) + .collect(Collectors.toList()); + } } } @@ -162,16 +269,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() .map(this::toVersion) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @@ -187,16 +292,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - checkVersion(tenantId, branch, versionId, path); - + checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); VersionedEntityInfo info = new VersionedEntityInfo(); info.setExternalId(entityId); @@ -205,171 +308,155 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @Override - public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); - } - - @Override - public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); - } - - @Override - public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, null, settings); - } + public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); - private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { List> entityDataList = new ArrayList<>(); - repositoryLock.readLock().lock(); + EntityVersion version; + repository.getLock().readLock().lock(); try { - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { - EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { + EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); entityDataList.add(entityData); } } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } EntityImportSettings importSettings = EntityImportSettings.builder() .updateRelations(settings.isLoadRelations()) .findExistingByName(settings.isFindExistingEntityByName()) .build(); + // FIXME [viacheslav]: do evrth in transaction List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - return importResults.stream() - .map(importResult -> EntityVersionLoadResult.builder() - .previousEntityVersion(importResult.getOldEntity()) - .newEntityVersion(importResult.getSavedEntity()) - .entityType(importResult.getEntityType()) - .build()) - .collect(Collectors.toList()); + Map results = new HashMap<>(); + + boolean removeNonexistentLocalEntities = false; + if () + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); + } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); + }); + } + + return new ArrayList<>(results.values()); } @SneakyThrows - private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { - repositoryLock.readLock().lock(); + private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); return JacksonUtil.fromString(entityDataJson, EntityExportData.class); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } - - private void fetch() throws GitAPIException { - repositoryLock.writeLock().lock(); - try { - repository.fetch(); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - private ScheduledFuture scheduleFetch(int fetchPeriod) { - return fetchExecutor.scheduleWithFixedDelay(() -> { - if (repository == null) return; - try { - fetch(); - } catch (Exception e) { - log.error("Failed to fetch remote repository", e); - } - }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { + ObjectNode versionInfo = JacksonUtil.newObjectNode(); + versionInfo.set("versionName", new TextNode(version.getName())); + versionInfo.set("versionId", new TextNode(version.getId())); + attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, + List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); } - private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Version not found"); - } - } - @Override - public List listAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptyList()); + public List listBranches(TenantId tenantId) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listBranches(); } - private void checkBranch(TenantId tenantId, String branch) { - if (!listAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to the branch"); - } - } + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); } - private void initRepository(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { - this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(Path.of(settings.getRepositoryDirectory())); - this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), - settings.getUsername(), settings.getPassword()); - } - } finally { - repositoryLock.writeLock().unlock(); + private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); } + repositories.put(tenantId, repository); } - private void clearRepository() throws IOException { - repositoryLock.writeLock().lock(); - try { - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repository = null; - } - } finally { - repositoryLock.writeLock().unlock(); + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); } } @SneakyThrows @Override - public void saveSettings(EntitiesVersionControlSettings settings) { - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .orElseGet(() -> { - AdminSettings newAdminSettings = new AdminSettings(); - newAdminSettings.setKey(SETTINGS_KEY); - return newAdminSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - - repositoryLock.writeLock().lock(); - try { - clearRepository(); - initRepository(settings); - } finally { - repositoryLock.writeLock().unlock(); - } + public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { + attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) + )).get(); - if (settings.getFetchPeriod() != 0) { - fetchTask.cancel(true); - fetchTask = scheduleFetch(settings.getFetchPeriod()); - } + clearRepository(tenantId); + initRepository(tenantId, settings); } + @SneakyThrows @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) .orElse(null); } @@ -387,7 +474,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 435c661b02..03343ec6f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -20,19 +20,18 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; import java.util.List; public interface EntitiesVersionControlService { - EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; - - EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; @@ -42,23 +41,19 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List listAllowedBranches(TenantId tenantId); + List listBranches(TenantId tenantId) throws Exception; - void saveSettings(EntitiesVersionControlSettings settings); + void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(); + EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 06e0a9c7aa..602f5dba4f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -17,18 +17,10 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; -import java.util.List; -import java.util.Map; -import java.util.UUID; - @Data public class EntitiesVersionControlSettings { - private String repositoryDirectory; private String repositoryUri; private String username; private String password; - - private int fetchPeriod; - - private Map> tenantsAllowedBranches; + private String defaultBranch; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java new file mode 100644 index 0000000000..ac13a7b8bf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; + +@Data +public class VersionCreationResult { + private EntityVersion version; + private int added; + private int modified; + private int removed; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java index 4cf0ee2b0f..cf8b588008 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java @@ -20,14 +20,14 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; @Data @AllArgsConstructor @NoArgsConstructor @Builder -public class EntityVersionLoadResult { - private ExportableEntity newEntityVersion; - private ExportableEntity previousEntityVersion; +public class VersionLoadResult { private EntityType entityType; + private int created; + private int updated; + private int deleted; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java index d71ddc8720..835f684021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java @@ -13,26 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityFilter; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityFilterExportRequest extends ExportRequest { +public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { private EntityFilter filter; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_FILTER; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java index 946676cf65..6386e9299a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java @@ -13,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityDataQuery; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityQueryExportRequest extends ExportRequest { +public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { private EntityDataQuery query; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_QUERY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java index 01309a421b..1f3e1e376f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java @@ -13,23 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityListExportRequest extends ExportRequest { +public class EntityListVersionCreateConfig extends VersionCreateConfig { private List entitiesIds; @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_LIST; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_LIST; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index fd2f659b70..6a8d59b876 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; @@ -21,18 +21,18 @@ import org.thingsboard.server.common.data.EntityType; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityTypeExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateConfig extends VersionCreateConfig { private EntityType entityType; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_TYPE; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_TYPE; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java index fe07ba7534..ab23f1175e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; -@EqualsAndHashCode(callSuper = true) @Data -public class SingleEntityExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateConfig extends VersionCreateConfig { private EntityId entityId; @Override - public ExportRequestType getType() { - return ExportRequestType.SINGLE_ENTITY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.SINGLE_ENTITY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index af0f161227..31b7b46f47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -23,17 +23,17 @@ import lombok.Data; @JsonTypeInfo(use = Id.NAME, property = "type") @JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = CustomEntityFilterExportRequest.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = CustomEntityQueryExportRequest.class) + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) }) @Data -public abstract class ExportRequest { +public abstract class VersionCreateConfig { - private EntityExportSettings exportSettings; + private boolean saveRelations; - public abstract ExportRequestType getType(); + public abstract VersionCreateConfigType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java index c36ac567e3..b2fb251d6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; -public enum ExportRequestType { +public enum VersionCreateConfigType { SINGLE_ENTITY, ENTITY_LIST, ENTITY_TYPE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 842f6ddd4e..47b07af21c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -13,12 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; +import java.util.List; + @Data -public class EntityVersionLoadSettings { - private boolean loadRelations; - private boolean findExistingEntityByName; +public class VersionCreateRequest { + + private String versionName; + private String branch; + + private List configs; + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java new file mode 100644 index 0000000000..f210dc9e0c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -0,0 +1,12 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { + + private boolean removeNonexistentLocalEntities; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java new file mode 100644 index 0000000000..6a49c45c2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadRequest extends VersionLoadRequest { + + private Map configs; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java new file mode 100644 index 0000000000..f7d99a0981 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class EntityVersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java new file mode 100644 index 0000000000..f5336b55d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionLoadRequest extends VersionLoadRequest { + + private EntityId externalEntityId; + + private EntityVersionLoadConfig config; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 1fc36463ba..660f5dc18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @Data -public class EntityVersionSaveSettings { - private boolean saveRelations; +public abstract class VersionLoadRequest { + + private String branch; + private String versionId; + + public abstract VersionLoadRequestType getType(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java new file mode 100644 index 0000000000..178d49668f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +public enum VersionLoadRequestType { + SINGLE_ENTITY, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 205bbca50e..14cdbf7b2c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -24,6 +24,7 @@ import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -33,18 +34,18 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -54,6 +55,8 @@ public class GitRepository { @Getter private final String directory; + @Getter + private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -61,21 +64,20 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(String uri, String directory, - String username, String password) throws GitAPIException { + public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() .setURI(uri) - .setDirectory(new java.io.File(directory)) + .setDirectory(directory) .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new GitRepository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); } - public static GitRepository open(String directory, String username, String password) throws IOException { - Git git = Git.open(new java.io.File(directory)); - return new GitRepository(git, newCredentialsProvider(username, password), directory); + public static GitRepository open(File directory, String username, String password) throws IOException { + Git git = Git.open(directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); } @@ -180,9 +182,17 @@ public class GitRepository { execute(git.clean()); } - - public Commit commit(String message, String filesPattern) throws GitAPIException { + public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); + } + + public Status status() throws GitAPIException { + org.eclipse.jgit.api.Status status = execute(git.status()); + return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + } + + public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); @@ -256,4 +266,11 @@ public class GitRepository { private final String authorName; } + @Data + public static class Status { + private final Set added; + private final Set modified; + private final Set removed; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index b5c5a5aab8..988bae7146 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -64,13 +64,13 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.nio.ByteBuffer; import java.util.Arrays; @@ -339,17 +339,17 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(entityId); exportRequest.setExportSettings(new EntityExportSettings()); return (EntityExportData) exportEntities(exportRequest).get(0); } - protected List> exportEntities(ExportRequest exportRequest) throws Exception { + protected List> exportEntities(VersionCreateConfig exportRequest) throws Exception { return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>() {}); } - protected List> exportEntities(List exportRequests) throws Exception { + protected List> exportEntities(List exportRequests) throws Exception { return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>() {}); } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index bd237e39d0..faa7f3a318 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; @@ -50,18 +49,18 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.ota.OtaPackageStateService; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.util.ArrayList; import java.util.List; @@ -294,11 +293,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp dashboard.setConfiguration(dashboardConfiguration); dashboard = dashboardService.saveDashboard(dashboard); - EntityTypeExportRequest assetsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig assetsExportRequest = new EntityTypeVersionCreateConfig(); assetsExportRequest.setEntityType(EntityType.ASSET); assetsExportRequest.setPageSize(10); assetsExportRequest.setExportSettings(new EntityExportSettings()); - EntityTypeExportRequest dashboardsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig dashboardsExportRequest = new EntityTypeVersionCreateConfig(); dashboardsExportRequest.setEntityType(EntityType.DASHBOARD); dashboardsExportRequest.setPageSize(10); dashboardsExportRequest.setExportSettings(new EntityExportSettings()); @@ -365,7 +364,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Customer 1 - Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), ruleChain.getId(), deviceProfile.getId(), dashboard.getId())); List> exportDataList = exportEntities(exportRequest); @@ -418,7 +417,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -464,7 +463,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -506,7 +505,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device1 = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(asset.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -538,7 +537,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(device.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -568,7 +567,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin1(); DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(defaultDeviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); List> exportDataList = exportEntities(exportRequest); @@ -613,7 +612,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testEntityTypeExportRequest(ExportableEntity entity) throws Exception { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entity.getId().getEntityType()); @@ -626,11 +625,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testCustomEntityFilterExportRequest(ExportableEntity entity) throws Exception { - CustomEntityFilterExportRequest exportRequest = new CustomEntityFilterExportRequest(); + EntitiesByCustomFilterVersionCreateConfig exportRequest = new EntitiesByCustomFilterVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); - EntityListFilter filter = new EntityListFilter(); + org.thingsboard.server.common.data.query.EntityListFilter filter = new org.thingsboard.server.common.data.query.EntityListFilter(); filter.setEntityType(entity.getId().getEntityType()); filter.setEntityList(List.of(entity.getId().toString())); exportRequest.setFilter(filter); @@ -652,10 +651,10 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Asset tenantAsset = createAsset(tenantId1, null, "A", "Tenant asset 1"); Asset customerAsset = createAsset(tenantId1, customer.getId(), "A", "Customer asset 1"); - List exportRequests = new ArrayList<>(); + List exportRequests = new ArrayList<>(); for (EntityType entityType : Set.of(EntityType.DEVICE, EntityType.ASSET)) { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entityType); @@ -685,7 +684,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), deviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 18444bb5f3..9240838543 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public abstract class DaoUtil { @@ -51,7 +53,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap()); } - public static Pageable toPageable(PageLink pageLink, Map columnMap) { + public static Pageable toPageable(PageLink pageLink, Map columnMap) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(pageLink.getSortOrder(), columnMap)); } @@ -59,7 +61,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } - public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(sortOrders, columnMap)); } @@ -108,4 +110,18 @@ public abstract class DaoUtil { return ids; } + public static void processInBatches(Function> finder, int batchSize, Consumer processor) { + PageLink pageLink = new PageLink(batchSize); + PageData batch; + + boolean hasNextBatch; + do { + batch = finder.apply(pageLink); + batch.getData().forEach(processor); + + hasNextBatch = batch.hasNext(); + pageLink = pageLink.nextPageLink(); + } while (hasNextBatch); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index cd534f24e6..aaa35e109a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -16,6 +16,8 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import java.util.UUID; @@ -25,4 +27,6 @@ public interface ExportableEntityDao> extends Dao< default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } + PageData findByTenantId(UUID tenantId, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index bcfdaae965..8879809ddb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -219,6 +219,11 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return findAssetsByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findAssetsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index c6ba310793..034431157a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -80,6 +80,11 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findCustomersByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 4a870b2a21..847314f1a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -31,4 +33,6 @@ public interface DashboardRepository extends JpaRepository findByTenantIdAndTitle(UUID tenantId, String title); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 6ff9fc6bfb..8fa0cbdc50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -21,6 +21,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -58,6 +60,11 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(dashboardRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + @Override public List findByTenantIdAndTitle(UUID tenantId, String title) { return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 93cd857660..d4663f60a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -313,6 +313,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return findDeviceByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findDevicesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index cae94cfb73..4cab209f49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -121,6 +121,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findDeviceProfiles(TenantId.fromUUID(tenantId), pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 0d5fea6f98..2703c99fba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -114,6 +114,11 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; From 2302213b3c8f288fa8284973e668285509f5e671 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 17:03:23 +0300 Subject: [PATCH 086/262] Refactor version create request structures --- .../DefaultEntitiesVersionControlService.java | 81 ++++++++++--------- ...tiesByCustomFilterVersionCreateConfig.java | 36 --------- ...itiesByCustomQueryVersionCreateConfig.java | 36 --------- .../create/EntityListVersionCreateConfig.java | 33 -------- .../EntityListVersionCreateRequest.java | 21 +++++ .../create/EntityTypeVersionCreateConfig.java | 38 --------- .../EntityTypeVersionCreateRequest.java | 20 +++++ .../MultipleEntitiesVersionCreateConfig.java | 10 +++ .../SingleEntityVersionCreateConfig.java | 33 -------- .../SingleEntityVersionCreateRequest.java | 19 +++++ .../vc/data/request/create/SyncStrategy.java | 6 ++ .../request/create/VersionCreateConfig.java | 16 ---- .../create/VersionCreateConfigType.java | 25 ------ .../request/create/VersionCreateRequest.java | 6 +- .../create/VersionCreateRequestType.java | 7 ++ .../load/EntityTypeVersionLoadConfig.java | 2 +- 16 files changed, 127 insertions(+), 262 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 919960ce62..f6434daecc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -65,6 +65,7 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; @@ -152,23 +153,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont if (repository.listBranches().contains(request.getBranch())) { repository.checkout(request.getBranch()); repository.merge(request.getBranch()); - } else { // FIXME [viacheslav]: rollback orphan branch on failure + } else { // TODO [viacheslav]: rollback orphan branch on failure repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - for (VersionCreateRequest.Config config : request.getConfigs()) { + for (VersionCreateConfig config : request.getConfigs()) { EntityExportSettings exportSettings = EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build(); List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities()) { + for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); entityDataList.add(entityData); } - if (config.isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() + if (config instanceof MultipleEntitiesVersionCreateConfig && + ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? .map(EntityExportData::getEntityType) .distinct() .forEach(entityType -> { @@ -186,7 +188,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } } - // TODO [viacheslav]: find with pagination repository.add("."); @@ -206,18 +207,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { - switch (entityFilter.getType()) { + private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { + switch (config.getType()) { case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; return List.of(filter.getEntityId()); } case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; return filter.getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); @@ -228,7 +229,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); EntityDataPageLink pageLink = new EntityDataPageLink(); @@ -243,7 +244,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() .stream().map(EntityData::getEntityId) @@ -341,35 +342,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont boolean removeNonexistentLocalEntities = false; if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); - - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); } - // need to delete in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); - } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); - } + } return new ArrayList<>(results.values()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java deleted file mode 100644 index 835f684021..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityFilter; - -import java.util.UUID; - -@Data -public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { - - private EntityFilter filter; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java deleted file mode 100644 index 6386e9299a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityDataQuery; - -import java.util.UUID; - -@Data -public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { - - private EntityDataQuery query; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java deleted file mode 100644 index 1f3e1e376f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -public class EntityListVersionCreateConfig extends VersionCreateConfig { - - private List entitiesIds; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java new file mode 100644 index 0000000000..fadd9b1fd9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -0,0 +1,21 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityListVersionCreateRequest extends VersionCreateRequest { + + private List entitiesIds; + private MultipleEntitiesVersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_LIST; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java deleted file mode 100644 index 6a8d59b876..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.EntityType; - -import java.util.UUID; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionCreateConfig extends VersionCreateConfig { - - private EntityType entityType; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_TYPE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java new file mode 100644 index 0000000000..0fd1d60667 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateRequest extends VersionCreateRequest { + + private Map entityTypes; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java new file mode 100644 index 0000000000..f6d852875d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -0,0 +1,10 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { + private SyncStrategy syncStrategy; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java deleted file mode 100644 index ab23f1175e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SingleEntityVersionCreateConfig extends VersionCreateConfig { - - private EntityId entityId; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.SINGLE_ENTITY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java new file mode 100644 index 0000000000..31bdeca658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -0,0 +1,19 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateRequest extends VersionCreateRequest { + + private EntityId entityId; + private VersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java new file mode 100644 index 0000000000..6216ad7658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum SyncStrategy { + MERGE, + OVERWRITE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 31b7b46f47..28f3232cb9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -15,25 +15,9 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import lombok.Data; -@JsonTypeInfo(use = Id.NAME, property = "type") -@JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) -}) @Data public abstract class VersionCreateConfig { - private boolean saveRelations; - - public abstract VersionCreateConfigType getType(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java deleted file mode 100644 index b2fb251d6a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -public enum VersionCreateConfigType { - SINGLE_ENTITY, - ENTITY_LIST, - ENTITY_TYPE, - - CUSTOM_ENTITY_FILTER, - CUSTOM_ENTITY_QUERY -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 47b07af21c..e11e2f85a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -17,14 +17,12 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import java.util.List; - @Data -public class VersionCreateRequest { +public abstract class VersionCreateRequest { private String versionName; private String branch; - private List configs; + public abstract VersionCreateRequestType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java new file mode 100644 index 0000000000..a5d0447b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -0,0 +1,7 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum VersionCreateRequestType { + SINGLE_ENTITY, + ENTITY_LIST, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index f210dc9e0c..b5ee5ddd50 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -7,6 +7,6 @@ import lombok.EqualsAndHashCode; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { - private boolean removeNonexistentLocalEntities; + private boolean removeOtherEntities; } From 5a43788b2498700135b4df4f6f1e5d7ed81b4f5c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 15:32:07 +0300 Subject: [PATCH 087/262] VC refactoring; Swagger examples --- .../EntitiesVersionControlController.java | 141 ++++++- .../DefaultEntitiesExportImportService.java | 40 +- .../EntitiesExportImportService.java | 5 + .../DefaultExportableEntitiesService.java | 46 --- .../exporting/ExportableEntitiesService.java | 2 - .../DefaultEntitiesVersionControlService.java | 386 +++++++++--------- .../sync/vc/data/VersionCreationResult.java | 15 + .../sync/vc/data/VersionedEntityInfo.java | 1 - .../EntityListVersionCreateRequest.java | 15 + .../EntityTypeVersionCreateRequest.java | 15 + .../MultipleEntitiesVersionCreateConfig.java | 17 +- .../SingleEntityVersionCreateRequest.java | 15 + .../vc/data/request/create/SyncStrategy.java | 15 + .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 9 + .../create/VersionCreateRequestType.java | 15 + .../load/EntityTypeVersionLoadConfig.java | 17 +- .../load/EntityTypeVersionLoadRequest.java | 17 +- .../request/load/EntityVersionLoadConfig.java | 11 - .../load/SingleEntityVersionLoadRequest.java | 17 +- .../data/request/load/VersionLoadConfig.java | 26 ++ .../data/request/load/VersionLoadRequest.java | 9 + .../request/load/VersionLoadRequestType.java | 15 + .../server/utils/GitRepository.java | 4 +- 24 files changed, 580 insertions(+), 275 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0625dbeb38..e7df5acdc3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -44,6 +45,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; + @RestController @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -53,6 +56,62 @@ public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"saveRelations\": true\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_LIST:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_LIST\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entitiesIds\": [\n" + + " {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " {\n" + + " \"entityType\": \"DEVICE_PROFILE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " }\n" + + " ],\n" + + " \"config\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " },\n" + + " \"DEVICE_PROFILE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"OVERWRITE\"\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/version") public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -64,6 +123,13 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profile 1 version 1.0\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") public List listEntityVersions(@PathVariable String branch, @PathVariable EntityType entityType, @@ -76,6 +142,13 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profiles from dev\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}") public List listEntityTypeVersions(@PathVariable String branch, @PathVariable EntityType entityType) throws ThingsboardException { @@ -86,6 +159,21 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" + + " \"name\": \"Device 1 removed\"\n" + + " },\n" + + " {\n" + + " \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \"name\": \"Device profiles added\"\n" + + " },\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Devices added\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}") public List listVersions(@PathVariable String branch) throws ThingsboardException { try { @@ -118,6 +206,38 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + " \n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \n" + + " \"externalEntityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false,\n" + + " \"removeOtherEntities\": true\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/entity") public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -139,8 +259,21 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + - "") + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"name\": \"master\",\n" + + " \"default\": true\n" + + " },\n" + + " {\n" + + " \"name\": \"dev\",\n" + + " \"default\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"dev-2\",\n" + + " \"default\": false\n" + + " }\n" + + "]\n\n```") @GetMapping("/branches") public List listBranches() throws ThingsboardException { try { @@ -161,7 +294,7 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + @@ -177,7 +310,7 @@ public class EntitiesVersionControlController extends BaseController { } } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 608a256514..108c2c0b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -70,15 +70,37 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } + @Override + public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + + EntityType entityType = exportData.getEntityType(); + EntityImportService> importService = getImportService(entityType); + + EntityImportResult importResult = importService.importEntity(user, exportData, importSettings); + + if (saveReferences) { + importResult.getSaveReferencesCallback().run(); + } + if (sendEvents) { + importResult.getSendEventsCallback().run(); + } + + return importResult; + } + @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixOrder(exportDataList); + fixDataOrderForImport(exportDataList); List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { - EntityImportResult importResult = importEntity(user, exportData, importSettings); + EntityImportResult importResult = importEntity(user, exportData, importSettings, false, false); importResults.add(importResult); } @@ -103,21 +125,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } - private void fixOrder(List> exportDataList) { + @Override + public void fixDataOrderForImport(List> exportDataList) { exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); } - private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { - if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { - throw new DataValidationException("Invalid entity data"); - } - - EntityType entityType = exportData.getEntityType(); - EntityImportService> importService = getImportService(entityType); - - return importService.importEntity(user, exportData, importSettings); - } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index 5c29feb57c..049cf4390a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -30,6 +30,11 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException; + List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; + void fixDataOrderForImport(List> exportDataList); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index 4d772ad935..dafd68ba7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -17,11 +17,9 @@ package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; @@ -47,12 +45,6 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -145,44 +137,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } - @Transactional(readOnly = true, timeout = 40) - @Override - public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { - switch (request.getType()) { - case SINGLE_ENTITY: { - return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); - } - case ENTITY_LIST: { - return ((EntityListVersionCreateConfig) request).getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(exportRequest.getEntityType()); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; - EntityFilter filter = exportRequest.getFilter(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; - EntityDataQuery query = exportRequest.getQuery(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByQuery(tenantId, customerId, query); - } - default: { - throw new IllegalArgumentException("Export request is not supported"); - } - } - } - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 043ec91e20..66c9079148 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -39,8 +39,6 @@ public interface ExportableEntitiesService { , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); - List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: - void deleteByTenantIdAndId(TenantId tenantId, I id); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index f6434daecc..0bdedb5e4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,16 +18,14 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -40,12 +38,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; @@ -65,17 +63,18 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; +import org.thingsboard.server.utils.ThrowingRunnable; import java.io.File; import java.io.IOException; @@ -108,10 +107,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final AttributesService attributesService; private final EntityService entityService; private final TenantDao tenantDao; + private final TransactionTemplate transactionTemplate; // TODO [viacheslav]: concurrency private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}") + @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; private static final String SETTINGS_KEY = "vc"; @@ -146,33 +146,26 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - repository.getLock().writeLock().lock(); - - try { - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } - - for (VersionCreateConfig config : request.getConfigs()) { - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(config.isSaveRelations()) - .build(); - List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); - } + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // TODO [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } - if (config instanceof MultipleEntitiesVersionCreateConfig && - ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? - .map(EntityExportData::getEntityType) - .distinct() + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + break; + } + case ENTITY_LIST: { + EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; + if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { + versionCreateRequest.getEntitiesIds().stream() + .map(EntityId::getEntityType).distinct() .forEach(entityType -> { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); @@ -181,76 +174,70 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } }); } - - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { + saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); } + break; } + case ENTITY_TYPE: { + EntityTypeVersionCreateRequest versionCreateRequest = (EntityTypeVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + break; + } + } - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); + repository.add("."); - result.setVersion(toVersion(commit)); - return result; - } finally { - repository.getLock().writeLock().unlock(); - } - } + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { - switch (config.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; - return List.of(filter.getEntityId()); - } - case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; - return filter.getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; - EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + GitRepository.Commit commit = repository.commit(request.getVersionName()); + repository.push(); - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(filter.getEntityType()); + result.setVersion(toVersion(commit)); + return result; + } - newFilter.setFilter(entityTypeFilter); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; - EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); - - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - newFilter.setQuery(query); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); - return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() - .stream().map(EntityData::getEntityId) - .collect(Collectors.toList()); - } - } + private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build()); + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } @@ -271,14 +258,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listVersions(TenantId tenantId, String branch, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); } @@ -294,23 +276,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); - - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - info.setEntityName(entityData.getEntity().getName()); - return info; - }) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); } @@ -318,81 +292,117 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - List> entityDataList = new ArrayList<>(); - EntityVersion version; - repository.getLock().readLock().lock(); - try { - version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { - EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); - entityDataList.add(entityData); - } - } finally { - repository.getLock().readLock().unlock(); - } - - EntityImportSettings importSettings = EntityImportSettings.builder() - .updateRelations(settings.isLoadRelations()) - .findExistingByName(settings.isFindExistingEntityByName()) - .build(); - // FIXME [viacheslav]: do evrth in transaction - List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - - Map results = new HashMap<>(); + EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - boolean removeNonexistentLocalEntities = false; - if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + result.getSaveReferencesCallback().run(); + return result; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + try { + importResult.getSendEventsCallback().run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + return List.of(VersionLoadResult.builder() + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } + case ENTITY_TYPE: { + EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; + return transactionTemplate.execute(status -> { + Map results = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + // order entity types.. + // or what + versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + AtomicInteger deleted = new AtomicInteger(); + + Set remoteEntities; + try { + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + .map(VersionedEntityInfo::getExternalId) + .collect(Collectors.toSet()); + for (EntityId externalEntityId : remoteEntities) { + EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (config.isRemoveOtherEntities()) { DaoUtil.processInBatches(pageLink -> { return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { try { exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); } catch (ThingsboardException e) { throw new RuntimeException(e); } - // need to delete in a specific order? + // need to delete entity types in a specific order? exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); deleted.getAndIncrement(); } }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); + } + + results.put(entityType, VersionLoadResult.builder() + .created(created.get()) + .updated(updated.get()) + .deleted(deleted.get()) + .build()); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return new ArrayList<>(results.values()); + }); } - - return new ArrayList<>(results.values()); - } - - @SneakyThrows - private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { - GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } finally { - repository.getLock().readLock().unlock(); + default: + throw new IllegalArgumentException("Unsupported version load request"); } } - private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { - ObjectNode versionInfo = JacksonUtil.newObjectNode(); - versionInfo.set("versionName", new TextNode(version.getName())); - versionInfo.set("versionId", new TextNode(version.getId())); - attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, - List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); + private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); + + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); } @@ -417,12 +427,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - if (Files.exists(repositoryDirectory)) { - repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - } + FileUtils.forceDelete(repositoryDirectory.toFile()); + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); repositories.put(tenantId, repository); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java index ac13a7b8bf..9194c67272 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 97bb6ada12..0a65297892 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -21,6 +21,5 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class VersionedEntityInfo { private EntityId externalId; - private String entityName; // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java index fadd9b1fd9..59cf7d0176 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java index 0fd1d60667..f53e563e28 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java index f6d852875d..0a38e3c922 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @@ -5,6 +20,6 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { +public class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { private SyncStrategy syncStrategy; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java index 31bdeca658..4d03404199 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java index 6216ad7658..43ec873f22 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum SyncStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 28f3232cb9..d4f8354df4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -18,6 +18,6 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @Data -public abstract class VersionCreateConfig { +public class VersionCreateConfig { private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index e11e2f85a3..f4ba122f6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateRequest.class) +}) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index a5d0447b40..add3591fa2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index b5ee5ddd50..27597f5e87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -5,7 +20,7 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { +public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java index 6a49c45c2c..0ce902c83e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ import java.util.Map; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadRequest extends VersionLoadRequest { - private Map configs; + private Map entityTypes; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java deleted file mode 100644 index f7d99a0981..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.thingsboard.server.service.sync.vc.data.request.load; - -import lombok.Data; - -@Data -public class EntityVersionLoadConfig { - - private boolean loadRelations; - private boolean findExistingEntityByName; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java index f5336b55d1..eb4bb1bdc1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ public class SingleEntityVersionLoadRequest extends VersionLoadRequest { private EntityId externalEntityId; - private EntityVersionLoadConfig config; + private VersionLoadConfig config; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java new file mode 100644 index 0000000000..6ead76f105 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class VersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 660f5dc18d..5a24bc78e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.load; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +import static com.fasterxml.jackson.annotation.JsonSubTypes.Type; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionLoadRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionLoadRequest.class) +}) @Data public abstract class VersionLoadRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java index 178d49668f..6a59cba676 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; public enum VersionLoadRequestType { diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 14cdbf7b2c..2cb01e6a8b 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -55,8 +55,6 @@ public class GitRepository { @Getter private final String directory; - @Getter - private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -88,7 +86,7 @@ public class GitRepository { public void checkout(String branch) throws GitAPIException { execute(git.checkout() - .setName(branch)); + .setName("origin/" + branch)); } public void merge(String branch) throws IOException, GitAPIException { From 9b209e407553509821ce8b3ad010d6624f3a1ea6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 18:00:53 +0300 Subject: [PATCH 088/262] Refactor version create request; fixes for git api usage --- .../EntitiesVersionControlController.java | 45 +++------ .../DefaultEntitiesVersionControlService.java | 96 ++++++++++--------- .../EntityListVersionCreateRequest.java | 36 ------- .../create/EntityTypeVersionCreateConfig.java | 4 +- .../request/create/VersionCreateRequest.java | 3 +- .../create/VersionCreateRequestType.java | 1 - .../server/utils/GitRepository.java | 30 +++--- 7 files changed, 86 insertions(+), 129 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..7c2c2ac483 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController { " \"saveRelations\": true\n" + " }\n" + "}\n```" + NEW_LINE + - "ENTITY_LIST:" + NEW_LINE + + "COMPLEX:" + NEW_LINE + "```\n{\n" + - " \"type\": \"ENTITY_LIST\",\n" + + " \"type\": \"COMPLEX\",\n" + "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + - "\n" + - " \"entitiesIds\": [\n" + - " {\n" + - " \"entityType\": \"DEVICE\",\n" + - " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + - " },\n" + - " {\n" + - " \"entityType\": \"DEVICE_PROFILE\",\n" + - " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + - " }\n" + - " ],\n" + - " \"config\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + - " }\n" + - "}\n```" + NEW_LINE + - "ENTITY_TYPE:" + NEW_LINE + - "```\n{\n" + - " \"type\": \"ENTITY_TYPE\",\n" + - "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + " \"entityTypes\": {\n" + " \"DEVICE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true\n" + " },\n" + " \"DEVICE_PROFILE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"OVERWRITE\"\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + " }\n" + " }\n" + "}\n```") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..7fa99d9e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; @@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; @@ -88,6 +90,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont repository.fetch(); if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); + repository.checkout("origin/" + request.getBranch(), false); + try { + repository.checkout(request.getBranch(), true); + } catch (RefAlreadyExistsException e) { + repository.checkout(request.getBranch(), false); + } repository.merge(request.getBranch()); } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + try { + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } } switch (request.getType()) { @@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } - case ENTITY_LIST: { - EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; - if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { - versionCreateRequest.getEntitiesIds().stream() - .map(EntityId::getEntityType).distinct() - .forEach(entityType -> { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { - saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); - } - break; - } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); } catch (IOException e) { @@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); - try { - saveEntityData(user, repository, entityId, config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + } catch (Exception e) { + throw new RuntimeException(e); + } } - }); + } + }); break; } @@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); @@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - clearRepository(tenantId); initRepository(tenantId, settings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java deleted file mode 100644 index adeac70e89..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityListVersionCreateRequest extends VersionCreateRequest { - - private List entitiesIds; - private EntityTypeVersionCreateConfig config; - - @Override - public VersionCreateRequestType getType() { - return VersionCreateRequestType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index 1c27d9d2f1..0cc890c0d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; import java.util.List; +import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) @@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig { //optional private SyncStrategy syncStrategy; - private List entityIds; + private List entityIds; private boolean allEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 10f093d26d..a99bd56d7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -23,8 +23,7 @@ import lombok.Data; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), - @Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class) + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) }) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index fea8b6fec7..aacc57f3b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, - ENTITY_LIST, COMPLEX } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..ac8df955d6 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -84,9 +81,15 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch) throws GitAPIException { + public void checkout(String branch, boolean createBranch) throws GitAPIException { execute(git.checkout() - .setName("origin/" + branch)); + .setCreateBranch(createBranch) + .setName(branch)); + } + + public void reset() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); } public void merge(String branch) throws IOException, GitAPIException { @@ -170,14 +173,15 @@ public class GitRepository { public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) + .setForced(true) .setName(name)); - Set uncommittedChanges = git.status().call().getUncommittedChanges(); - if (!uncommittedChanges.isEmpty()) { - RmCommand rm = git.rm(); - uncommittedChanges.forEach(rm::addFilepattern); - execute(rm); - } - execute(git.clean()); +// Set uncommittedChanges = git.status().call().getUncommittedChanges(); +// if (!uncommittedChanges.isEmpty()) { +// RmCommand rm = git.rm(); +// uncommittedChanges.forEach(rm::addFilepattern); +// execute(rm); +// } +// execute(git.clean()); } public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] From 7ec45fd5f1d602f23fde6225b7c4ee7b77e66f52 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 10:40:02 +0300 Subject: [PATCH 089/262] Rename Data structures. Create separate module. --- application/pom.xml | 4 + .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesVersionControlController.java | 14 +-- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 20 ++-- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 2 +- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 5 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 3 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +-- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../DefaultEntitiesVersionControlService.java | 42 ++++----- .../vc/EntitiesVersionControlService.java | 14 +-- .../common/data/sync}/JsonTbEntity.java | 2 +- .../common/data/sync}/ThrowingRunnable.java | 2 +- .../data/sync/ie}/DeviceExportData.java | 2 +- .../data/sync/ie}/EntityExportData.java | 4 +- .../data/sync/ie}/EntityExportSettings.java | 2 +- .../data/sync/ie}/EntityImportResult.java | 6 +- .../data/sync/ie}/EntityImportSettings.java | 2 +- .../data/sync/ie}/RuleChainExportData.java | 2 +- .../vc}/EntitiesVersionControlSettings.java | 2 +- .../common/data/sync/vc}/EntityVersion.java | 2 +- .../data/sync/vc}/VersionCreationResult.java | 2 +- .../data/sync/vc}/VersionLoadResult.java | 2 +- .../data/sync/vc}/VersionedEntityInfo.java | 2 +- .../create/ComplexVersionCreateRequest.java | 2 +- .../create/EntityTypeVersionCreateConfig.java | 2 +- .../SingleEntityVersionCreateRequest.java | 2 +- .../sync/vc}/request/create/SyncStrategy.java | 2 +- .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 2 +- .../create/VersionCreateRequestType.java | 2 +- .../load/EntityTypeVersionLoadConfig.java | 2 +- .../load/EntityTypeVersionLoadRequest.java | 2 +- .../load/SingleEntityVersionLoadRequest.java | 2 +- .../vc}/request/load/VersionLoadConfig.java | 2 +- .../vc}/request/load/VersionLoadRequest.java | 2 +- .../request/load/VersionLoadRequestType.java | 2 +- common/pom.xml | 1 + common/version-control/pom.xml | 94 +++++++++++++++++++ pom.xml | 5 + 61 files changed, 241 insertions(+), 141 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/DefaultEntitiesExportImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/EntitiesExportImportService.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/DefaultExportableEntitiesService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/ExportableEntitiesService.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/AbstractBulkImportService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/RuleChainImportService.java (95%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/JsonTbEntity.java (97%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/ThrowingRunnable.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/DeviceExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportResult.java (89%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/RuleChainExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntitiesVersionControlSettings.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntityVersion.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionCreationResult.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionLoadResult.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionedEntityInfo.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/ComplexVersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/EntityTypeVersionCreateConfig.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SingleEntityVersionCreateRequest.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SyncStrategy.java (90%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateConfig.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequestType.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/SingleEntityVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequestType.java (91%) create mode 100644 common/version-control/pom.xml diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..2ff2204b5a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 01401deebe..2a3268146f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,8 +51,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 61e44241d7..8b6f49c491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,8 +66,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8136b6d052..b3bbb03f4c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 7c2c2ac483..2ab75c565d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -32,13 +32,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index a2b71267b4..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,8 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index ee119c4dce..f0a7b6231e 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fea50538cc..b0fb2161e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,8 +28,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 108c2c0b1b..72e3d757be 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,15 +27,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 049cf4390a..5f5bed9dac 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index dafd68ba7b..18c5f5c7c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 5532f30125..9b251a3dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 66c9079148..742b993cfc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -25,9 +25,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; - -import java.util.List; public interface ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 46428a763f..80850cb462 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 69af613480..31bf768e90 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 93f9c51655..2e03b92a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index f40ee6884a..15c9c72a74 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index be6f62efe4..bf249937b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing; +package org.thingsboard.server.service.sync.ie.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 2be56ebe2a..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -66,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 9c0e6b6491..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 6347910cab..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 550d4a72f3..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index a39dcc7481..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 80a942c73f..fe6c3f1875 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ecfa7d3ab..203b0caa16 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index b69c5a0d5b..2a7570e644 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index b85e403f87..16ebdd580e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 112152365b..eb123a63af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index f97950aaf8..592fe4bc5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 993f9fa27a..1f1f15a74e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7fa99d9e6d..c22eb6a5a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -55,28 +55,28 @@ import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; import java.io.IOException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..e025f80fed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,13 +19,13 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index cfc38ff3c3..9d326246d4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; import com.fasterxml.jackson.annotation.JsonSubTypes; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java index 858ccdf422..1cb2ac8c74 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import org.thingsboard.server.common.data.exception.ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index 4be338d7cc..faf9280b93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 81e3b344d0..b361a302fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.utils.JsonTbEntity; +import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0fb318b5fb..051745c07f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 242856bfcb..ea69425a2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.utils.JsonTbEntity; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.JsonTbEntity; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 075a6896ef..3e18bc9f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java index e1d870473f..b176782f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ee8f5b40b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 3547f89e3c..8779d64ed3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java index 9194c67272..8cb09b1b30 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index cf8b588008..0039546fb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 0a65297892..163fe4c6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java index 3c34680f95..f8d7279f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java index 0cc890c0d7..92cab96354 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java index 4d03404199..507afd116e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java index 43ec873f22..baf24efeb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum SyncStrategy { MERGE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index d4f8354df4..a1aa6cd6d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java index a99bd56d7f..9a23838921 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java index aacc57f3b3..d57363cf7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 27597f5e87..837986f82d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java index 0ce902c83e..f2cb83f0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java index eb4bb1bdc1..cf31317b4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 6ead76f105..31b34dbcb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java index 5a24bc78e5..d9d1329c8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java index 6a59cba676..190e6871d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; public enum VersionLoadRequestType { SINGLE_ENTITY, diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..1b7194e029 --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.springframework + spring-core + + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.thingsboard.common + data + + + + + + + + + diff --git a/pom.xml b/pom.xml index e215e0719d..324de28f3a 100755 --- a/pom.xml +++ b/pom.xml @@ -866,6 +866,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache From c6ebf049a2dd7910293a3d728533686ec6bc663a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 14:46:54 +0300 Subject: [PATCH 090/262] Version Control Service refactoring --- application/pom.xml | 4 - .../EntitiesVersionControlController.java | 2 +- .../DefaultEntitiesVersionControlService.java | 211 ++------------ .../vc/EntitiesVersionControlService.java | 2 +- .../vc/LocalGitVersionControlService.java | 258 ++++++++++++++++++ .../src/main/resources/thingsboard.yml | 5 + common/version-control/pom.xml | 21 ++ .../sync/vc/DefaultGitRepositoryService.java | 239 ++++++++++++++++ .../service/sync/vc}/GitRepository.java | 2 +- .../service/sync/vc/GitRepositoryService.java | 48 ++++ .../sync/vc/GitVersionControlService.java | 60 ++++ .../server/service/sync/vc/PendingCommit.java | 36 +++ 12 files changed, 695 insertions(+), 193 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java rename {application/src/main/java/org/thingsboard/server/utils => common/version-control/src/main/java/org/thingsboard/server/service/sync/vc}/GitRepository.java (99%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java diff --git a/application/pom.xml b/application/pom.xml index 2ff2204b5a..37d782b3b0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,10 +345,6 @@ Java-WebSocket test - - org.eclipse.jgit - org.eclipse.jgit - diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 2ab75c565d..2794db90d4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController { @PathVariable EntityType entityType, @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c22eb6a5a5..e222d027e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.utils.GitRepository; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; @@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final GitVersionControlService gitService; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AttributesService attributesService; private final EntityService entityService; - private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; - // TODO [viacheslav]: concurrency - private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}/repositories") - private String repositoriesFolder; - - private static final String SETTINGS_KEY = "vc"; - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); - if (settings != null) { - try { - initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, 5, 5, TimeUnit.SECONDS); - } - + public static final String SETTINGS_KEY = "vc"; @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout("origin/" + request.getBranch(), false); - try { - repository.checkout(request.getBranch(), true); - } catch (RefAlreadyExistsException e) { - repository.checkout(request.getBranch(), false); - } - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } - } + + var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } + gitService.deleteAll(commit, entityType); } if (config.isAllEntities()) { @@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, 100, data -> { EntityId entityId = data.getEntityId(); try { - saveEntityData(user, repository, entityId, config); + saveEntityData(user, commit, entityId, config); } catch (Exception e) { throw new RuntimeException(e); } @@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); - - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); - - result.setVersion(toVersion(commit)); - return result; + return gitService.push(commit); } - private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + gitService.addToCommit(commit, entityData); } @Override public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + return gitService.listVersions(tenantId, branch, externalId); } @Override public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + return gitService.listVersions(tenantId, branch, entityType); } @Override public List listVersions(TenantId tenantId, String branch) throws Exception { - return listVersions(tenantId, branch, null); - } - - private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return gitService.listVersions(tenantId, branch); } - @Override - public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, null); + return gitService.listEntitiesAtVersion(tenantId, branch, versionId); } - private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - return info; - }) - .collect(Collectors.toList()); - } - - @Override public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); result.getSaveReferencesCallback().run(); return result; } catch (Exception e) { @@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Set remoteEntities; try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() .map(VersionedEntityInfo::getExternalId) .collect(Collectors.toSet()); for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); if (importResult.getOldEntity() == null) created.incrementAndGet(); else updated.incrementAndGet(); @@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); - + private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .findExistingByName(config.isFindExistingEntityByName()) @@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listBranches(TenantId tenantId) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listBranches(); - } - - - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - - private GitRepository checkRepository(TenantId tenantId) { - return Optional.ofNullable(repositories.get(tenantId)) - .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); - } - - private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - if (Files.exists(repositoryDirectory)) { - FileUtils.forceDelete(repositoryDirectory.toFile()); - } - - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - repositories.put(tenantId, repository); - } - - private void clearRepository(TenantId tenantId) throws IOException { - GitRepository repository = repositories.get(tenantId); - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repositories.remove(tenantId); - } + return gitService.listBranches(tenantId); } - @SneakyThrows @Override public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { @@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - initRepository(tenantId, settings); + gitService.initRepository(tenantId, settings); } - @SneakyThrows @Override public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - - private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; + return gitService.getSettings(tenantId); } - - private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); - String entityId = StringUtils.substringBetween(path, "/", ".json"); - return EntityIdFactory.getByTypeAndUuid(entityType, entityId); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index e025f80fed..f0d4169b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -41,7 +41,7 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java new file mode 100644 index 0000000000..b9a5a9260b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -0,0 +1,258 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.io.IOException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; + +@Slf4j +@RequiredArgsConstructor +@Service +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +public class LocalGitVersionControlService implements GitVersionControlService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final GitRepositoryService gitRepositoryService; + private final TenantDao tenantDao; + private final AttributesService attributesService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + @AfterStartUp + public void init() { + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } + } + }); + } + + @Override + @SneakyThrows + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) + .orElse(null); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + var pendingCommit = new PendingCommit(tenantId, request); + PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); + if (old != null) { + gitRepositoryService.abort(old); + } + gitRepositoryService.prepareCommit(pendingCommit); + return pendingCommit; + } finally { + lock.unlock(); + } + } + + @Override + public void deleteAll(PendingCommit commit, EntityType entityType) { + doInsideLock(commit, c -> { + try { + gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public void addToCommit(PendingCommit commit, EntityExportData> entityData) { + doInsideLock(commit, c -> { + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString()), entityDataJson); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + return executeInsideLock(commit, gitRepositoryService::push); + } + + @Override + public List listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, branch, (String) null); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + try { + return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + @Override + public List listBranches(TenantId tenantId) { + return gitRepositoryService.listBranches(tenantId); + } + + @Override + public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { + try { + String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, + getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private List listVersions(TenantId tenantId, String branch, String path) { + try { + return gitRepositoryService.listVersions(tenantId, branch, path); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private void doInsideLock(PendingCommit commit, Consumer r) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + r.accept(commit); + } finally { + lock.unlock(); + } + } + + private T executeInsideLock(PendingCommit commit, Function c) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + return c.apply(commit); + } finally { + lock.unlock(); + } + } + + private void checkCommit(PendingCommit commit) { + PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); + if (existing == null || !existing.getTxId().equals(commit.getTxId())) { + throw new ConcurrentModificationException(); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03dbb49763..654929f500 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1115,6 +1115,11 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + git: + service: "${JS_VC_GIT_SERVICE:local}" # local/remote + repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + management: endpoints: web: diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 1b7194e029..f29b7db81a 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -40,6 +40,23 @@ org.springframework spring-core + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + com.google.guava guava @@ -65,6 +82,10 @@ ch.qos.logback logback-classic + + org.eclipse.jgit + org.eclipse.jgit + org.springframework.boot spring-boot-starter-test diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..d22fcf5d94 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + @Value("${vc.git.repos-poll-interval:60}") + private long reposPollInterval; + + private ScheduledExecutorService scheduler; + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getRequest().getBranch(); + try { + repository.fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout("origin/" + branch, false); + try { + repository.checkout(branch, true); + } catch (RefAlreadyExistsException e) { + repository.checkout(branch, false); + } + repository.merge(branch); + } else { // TODO [viacheslav]: rollback orphan branch on failure + try { + repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); + repository.push(); + + result.setVersion(toVersion(gitCommit)); + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void abort(PendingCommit commit) { + //TODO: implement; + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + } + + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + } + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/utils/GitRepository.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index ac8df955d6..afebe1f7fc 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..2191f38930 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.io.IOException; +import java.util.List; + +public interface GitRepositoryService { + + void prepareCommit(PendingCommit pendingCommit); + + List listVersions(TenantId tenantId, String branch, String path) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java new file mode 100644 index 0000000000..e67a8cca5f --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlService { + + @SneakyThrows + EntitiesVersionControlSettings getSettings(TenantId tenantId); + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); + + void addToCommit(PendingCommit commit, EntityExportData> entityData); + + void deleteAll(PendingCommit pendingCommit, EntityType entityType); + + VersionCreationResult push(PendingCommit commit); + + List listVersions(TenantId tenantId, String branch); + + List listVersions(TenantId tenantId, String branch, EntityType entityType); + + List listVersions(TenantId tenantId, String branch, EntityId entityId); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + List listBranches(TenantId tenantId); + + EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..a30f7514a2 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final TenantId tenantId; + private final VersionCreateRequest request; + + public PendingCommit(TenantId tenantId, VersionCreateRequest request) { + this.txId = UUID.randomUUID(); + this.tenantId = tenantId; + this.request = request; + } +} From 1d6b9a5cbdc05c514787e6c6d795dc66549fcb73 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 16:43:01 +0300 Subject: [PATCH 091/262] Version control executor --- common/version-control/pom.xml | 27 ++++ .../sync/vc/DefaultGitRepositoryService.java | 1 - .../version-control/src/main/proto/vc.proto | 71 +++++++++ msa/pom.xml | 1 + msa/vc-executor/pom.xml | 136 ++++++++++++++++++ msa/vc-executor/src/main/conf/logback.xml | 43 ++++++ .../src/main/conf/tb-vc-executor.conf | 22 +++ ...oardVersionControlExecutorApplication.java | 47 ++++++ .../src/main/resources/logback.xml | 34 +++++ .../src/main/resources/tb-vc-executor.yml | 27 ++++ 10 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 common/version-control/src/main/proto/vc.proto create mode 100644 msa/vc-executor/pom.xml create mode 100644 msa/vc-executor/src/main/conf/logback.xml create mode 100644 msa/vc-executor/src/main/conf/tb-vc-executor.conf create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java create mode 100644 msa/vc-executor/src/main/resources/logback.xml create mode 100644 msa/vc-executor/src/main/resources/tb-vc-executor.yml diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index f29b7db81a..164640fe73 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,6 +86,21 @@ org.eclipse.jgit org.eclipse.jgit + + io.grpc + grpc-netty-shaded + provided + + + io.grpc + grpc-protobuf + provided + + + io.grpc + grpc-stub + provided + org.springframework.boot spring-boot-starter-test @@ -109,7 +124,19 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index d22fcf5d94..c93b4ae76d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -226,7 +226,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto new file mode 100644 index 0000000000..e8c7e09384 --- /dev/null +++ b/common/version-control/src/main/proto/vc.proto @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +option java_package = "org.thingsboard.server.gen.vc.v1"; +option java_multiple_files = true; +option java_outer_classname = "EdgeProtos"; + +package vc; + +// Interface exported by the ThingsBoard Core. +service TbGitRpcService { + + rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} + +} + + +/** + * Data Structures; + */ +message CommitRequestMsg { + string txId = 1; + PrepareMsg prepareMsg = 2; + AddMsg addMsg = 3; + DeleteMsg deleteMsg = 4; + PushMsg pushMsg = 5; + AbortMsg abortMsg = 6; +} + +message CommitResponseMsg { + string id = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string tenantId = 1; + string commitMsg = 2; + string branchName = 3; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} \ No newline at end of file diff --git a/msa/pom.xml b/msa/pom.xml index 106f2be2a3..888291502b 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -40,6 +40,7 @@ tb + vc-executor js-executor web-ui tb-node diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml new file mode 100644 index 0000000000..7e4df36117 --- /dev/null +++ b/msa/vc-executor/pom.xml @@ -0,0 +1,136 @@ + + + 4.0.0 + + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor + + ThingsBoard Version Control Executor + https://thingsboard.io + Project for ThingsBoard version control microservice + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-mqtt-transport + false + ${project.build.directory}/windows + ThingsBoard Version Control Executor Service + org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication + + + + + org.thingsboard.common + version-control + + + org.springframework.boot + spring-boot-starter-web + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml new file mode 100644 index 0000000000..d62cf2b3f5 --- /dev/null +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -0,0 +1,43 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/msa/vc-executor/src/main/conf/tb-vc-executor.conf b/msa/vc-executor/src/main/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..83287286bb --- /dev/null +++ b/msa/vc-executor/src/main/conf/tb-vc-executor.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java new file mode 100644 index 0000000000..2724a566ee --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.vc; /** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +public class ThingsboardVersionControlExecutorApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-vc-executor"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardVersionControlExecutorApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml new file mode 100644 index 0000000000..2834934ee0 --- /dev/null +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -0,0 +1,34 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml new file mode 100644 index 0000000000..1de47e2750 --- /dev/null +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# If you enabled process metrics you should also enable 'web-environment'. +spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" +# If you enabled process metrics you should set 'web-application-type' to 'servlet' value. +spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" + +server: + # Server bind address (has no effect if web-environment is disabled). + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port (has no effect if web-environment is disabled). + port: "${HTTP_BIND_PORT:8080}" + From 567ee5d75bb20b9a2803adbd506b5a8e23dfc450 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 18 May 2022 18:49:21 +0300 Subject: [PATCH 092/262] Refactor loadEntitiesVersion --- .../DefaultEntitiesExportImportService.java | 12 +- .../sync/ie/EntitiesExportImportService.java | 7 +- .../DefaultEntitiesVersionControlService.java | 175 +++++++----------- .../vc/LocalGitVersionControlService.java | 9 + .../sync/vc/GitVersionControlService.java | 5 +- 5 files changed, 95 insertions(+), 113 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 72e3d757be..38d3b00213 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -95,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixDataOrderForImport(exportDataList); + exportDataList.sort(getDataComparatorForImport()); List> importResults = new ArrayList<>(); @@ -125,9 +125,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + + @Override + public Comparator> getDataComparatorForImport() { + return Comparator.comparing(EntityExportData::getEntityType, getEntityTypeComparatorForImport()); + } + @Override - public void fixDataOrderForImport(List> exportDataList) { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 5f5bed9dac..7bdb271444 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -24,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import java.util.Comparator; import java.util.List; public interface EntitiesExportImportService { @@ -35,6 +37,9 @@ public interface EntitiesExportImportService { List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; - void fixDataOrderForImport(List> exportDataList); + + Comparator> getDataComparatorForImport(); + + Comparator getEntityTypeComparatorForImport(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index e222d027e7..f6319db9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,18 +15,10 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; @@ -34,24 +26,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -77,27 +60,16 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; - @Service @TbCoreComponent @RequiredArgsConstructor @@ -115,7 +87,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { @@ -132,23 +103,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } if (config.isAllEntities()) { - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { try { - saveEntityData(user, commit, entityId, config); + saveEntityData(user, commit, entity.getId(), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -209,22 +168,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); - result.getSaveReferencesCallback().run(); - return result; + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); } }); - try { - importResult.getSendEventsCallback().run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); - } - return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) .updated(importResult.getOldEntity() != null ? 1 : 0) .deleted(0) @@ -234,56 +191,68 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; return transactionTemplate.execute(status -> { Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); List sendEventsCallbacks = new ArrayList<>(); - // order entity types.. - // or what - versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); - AtomicInteger deleted = new AtomicInteger(); - - Set remoteEntities; - try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() - .map(VersionedEntityInfo::getExternalId) - .collect(Collectors.toSet()); - for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - if (config.isRemoveOtherEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - // need to delete entity types in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); + versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()) + .forEach(entityType -> { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + try { + int limit = 100; + int offset = 0; + List> entityDataList; + do { + entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + for (EntityExportData entityData : entityDataList) { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + } catch (Exception e) { + throw new RuntimeException(e); } + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); }); - } - results.put(entityType, VersionLoadResult.builder() - .created(created.get()) - .updated(updated.get()) - .deleted(deleted.get()) - .build()); - }); + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + VersionLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); + } + }); + }); for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { @@ -307,14 +276,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - } - @Override public List listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index b9a5a9260b..cd96f5a6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -52,6 +52,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @RequiredArgsConstructor @@ -193,6 +194,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { return gitRepositoryService.listBranches(tenantId); } + @Override + public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { + return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() + .skip(offset).limit(limit) + .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) + .collect(Collectors.toList()); + } + @Override public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { try { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index e67a8cca5f..35d68347b1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import lombok.SneakyThrows; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,7 +30,6 @@ import java.util.List; public interface GitVersionControlService { - @SneakyThrows EntitiesVersionControlSettings getSettings(TenantId tenantId); void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); @@ -57,4 +55,7 @@ public interface GitVersionControlService { List listBranches(TenantId tenantId); EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); + + List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); + } From 0b6de71768c6ff356224c177b222d5b50a1e66d1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:01:06 +0300 Subject: [PATCH 093/262] Implement version control ssh access with private key. Improve VC settings REST methods. Add tenantId field to admin settings. Add DB upgrade. --- application/pom.xml | 4 + .../main/data/upgrade/3.3.4/schema_update.sql | 3 + .../server/controller/AdminController.java | 61 +++++++- .../EntitiesVersionControlController.java | 47 +----- .../install/ThingsboardInstallService.java | 1 + .../fetch/AdminSettingsEdgeEventFetcher.java | 7 +- .../DefaultSystemDataLoaderService.java | 2 + .../install/SqlDatabaseUpgradeService.java | 12 ++ .../permission/TenantAdminPermissions.java | 1 + .../system/DefaultSystemSecurityService.java | 1 + .../DefaultEntitiesVersionControlService.java | 145 ++++++++++-------- .../vc/EntitiesVersionControlService.java | 12 +- .../server/utils/GitRepository.java | 131 +++++++++++++--- .../server/common/data/AdminSettings.java | 19 ++- .../server/common/data/StringUtils.java | 42 +++++ .../vc}/EntitiesVersionControlSettings.java | 6 +- .../data/vc/VersionControlAuthMethod.java | 21 +++ .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AdminSettingsEntity.java | 9 ++ .../server/dao/settings/AdminSettingsDao.java | 4 +- .../settings/AdminSettingsServiceImpl.java | 22 ++- .../sql/settings/AdminSettingsRepository.java | 3 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 4 +- .../main/resources/sql/schema-entities.sql | 1 + pom.xml | 5 + 25 files changed, 411 insertions(+), 154 deletions(-) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/vc}/EntitiesVersionControlSettings.java (79%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..83d36c2b42 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,6 +345,10 @@ org.eclipse.jgit org.eclipse.jgit + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index ad307a6576..f537acfb36 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -26,3 +26,6 @@ ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE customer ADD COLUMN IF NOT EXISTS external_id UUID; + +ALTER TABLE admin_settings + ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index e5934cbf26..3868476491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -20,12 +20,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; @@ -34,14 +29,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @@ -60,6 +58,9 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private EntitiesVersionControlService versionControlService; + @Autowired private UpdateService updateService; @@ -96,6 +97,7 @@ public class AdminController extends BaseController { @RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + adminSettings.setTenantId(getTenantId()); adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); @@ -180,6 +182,53 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get version control settings (getVersionControlSettings)", + notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings") + @ResponseBody + public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vsSettings") + public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + versionControlService.saveVersionControlSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", + notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) + public void checkVersionControlAccess( + @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") + @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + versionControlService.checkVersionControlAccess(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..599149402f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,25 +15,18 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; @@ -280,7 +273,7 @@ public class EntitiesVersionControlController extends BaseController { List remoteBranches = versionControlService.listBranches(getTenantId()); List infos = new ArrayList<>(); - String defaultBranch = getSettings().getDefaultBranch(); + String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); @@ -293,40 +286,6 @@ public class EntitiesVersionControlController extends BaseController { } } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @GetMapping("/settings") - public EntitiesVersionControlSettings getSettings() throws ThingsboardException { - try { - return versionControlService.getSettings(getTenantId()); - } catch (Exception e) { - throw handleException(e); - } - } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @PostMapping("/settings") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { - try { - versionControlService.saveSettings(getTenantId(), settings); - } catch (Exception e) { - throw handleException(e); - } - } - - @Data public static class BranchInfo { private final String name; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ebab403149..e728c004c0 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -219,6 +219,7 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.3.3"); case "3.3.4": log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java index 0f4ecf7e30..ee10c7859b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java @@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings))); - AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); + AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings))); @@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates))); - AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); + AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates))); @@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { } } - private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) { + private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) { AdminSettings tenantMailSettings = new AdminSettings(); + tenantMailSettings.setTenantId(tenantId); jsonValue.put("useSystemMailSettings", true); tenantMailSettings.setJsonValue(jsonValue); tenantMailSettings.setKey(key); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 9e7636879f..696ac50c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -231,6 +231,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override public void createAdminSettings() throws Exception { AdminSettings generalSettings = new AdminSettings(); + generalSettings.setTenantId(TenantId.SYS_TENANT_ID); generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); @@ -239,6 +240,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); AdminSettings mailSettings = new AdminSettings(); + mailSettings.setTenantId(TenantId.SYS_TENANT_ID); mailSettings.setKey("mail"); node = objectMapper.createObjectNode(); node.put("mailFrom", "ThingsBoard "); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index f7b9e5cad1..1f9dcf15a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -534,6 +534,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 4ff884f0e1..8f821f5051 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -28,6 +28,7 @@ public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); + put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); put(Resource.ASSET, tenantEntityPermissionChecker); put(Resource.DEVICE, tenantEntityPermissionChecker); diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 9a998544f3..2449d7fbc0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -107,6 +107,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService { AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, "securitySettings"); if (adminSettings == null) { adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); adminSettings.setKey("securitySettings"); } adminSettings.setJsonValue(JacksonUtil.valueToTree(securitySettings)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..77a5586a8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -19,34 +19,29 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.*; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -58,17 +53,11 @@ import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExp import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.*; import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; @@ -81,13 +70,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -104,7 +87,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AttributesService attributesService; + private final AdminSettingsService adminSettingsService; private final EntityService entityService; private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; @@ -114,14 +97,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; - private static final String SETTINGS_KEY = "vc"; + private static final String SETTINGS_KEY = "entitiesVersionControl"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { initRepository(tenantId, settings); @@ -412,6 +395,73 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return repository.listBranches(); } + @Override + public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + + @Override + public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); + AdminSettings adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + try { + clearRepository(tenantId); + initRepository(tenantId, savedVersionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } + return savedVersionControlSettings; + } + + @Override + public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + settings = this.restoreCredentials(settings, storedSettings); + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + try { + GitRepository.test(settings, repositoryDirectory.toFile()); + } catch (GitAPIException e) { + throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), + ThingsboardErrorCode.GENERAL); + } + } + + private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + if (storedSettings != null) { + settings.setPassword(storedSettings.getPassword()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + if (storedSettings != null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && + StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { return listVersions(tenantId, branch, null).stream() @@ -426,11 +476,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + FileUtils.forceDelete(repositoryDirectory.toFile()); Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + + GitRepository repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); } @@ -442,34 +493,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - - @SneakyThrows - @Override - public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { - attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) - )).get(); - - clearRepository(tenantId); - initRepository(tenantId, settings); - } - - @SneakyThrows - @Override - public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..2854ec1d05 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,16 +16,17 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import java.util.List; @@ -51,9 +52,12 @@ public interface EntitiesVersionControlService { List listBranches(TenantId tenantId) throws Exception; + EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + + EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; - void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..e207c30fa2 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -18,14 +18,8 @@ package org.thingsboard.server.utils; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; -import org.eclipse.jgit.api.TransportCommand; +import org.apache.sshd.common.util.security.SecurityUtils; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -34,50 +28,88 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.sshd.JGitKeyCache; +import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; +import org.eclipse.jgit.transport.sshd.SshdSessionFactory; +import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PublicKey; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; + private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; + this.sshSessionFactory = sshSessionFactory; this.directory = directory; } - public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { - CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); - Git git = Git.cloneRepository() - .setURI(uri) + public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(settings.getRepositoryUri()) .setDirectory(directory) - .setNoCheckout(true) - .setCredentialsProvider(credentialsProvider) - .call(); - return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); + .setNoCheckout(true); + configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory); + Git git = cloneCommand.call(); + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, String username, String password) throws IOException { + public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { Git git = Git.open(directory); - return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } + public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); + configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory); + lsRemoteCommand.call(); + } public void fetch() throws GitAPIException { execute(git.fetch() @@ -108,7 +140,6 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { return listCommits(branch, null, limit); } @@ -246,16 +277,66 @@ public class GitRepository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand && credentialsProvider != null) { - ((TransportCommand) command).setCredentialsProvider(credentialsProvider); + if (command instanceof TransportCommand) { + configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory); } return command.call(); } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { + if (credentialsProvider != null) { + transportCommand.setCredentialsProvider(credentialsProvider); + } + if (sshSessionFactory != null) { + transportCommand.setTransportConfigCallback(transport -> { + if (transport instanceof SshTransport) { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + } + private static CredentialsProvider newCredentialsProvider(String username, String password) { - return new UsernamePasswordCredentialsProvider(username, password); + return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password); } + private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) { + SshdSessionFactory sshSessionFactory = null; + if (StringUtils.isNotBlank(privateKey)) { + Iterable keyPairs = loadKeyPairs(privateKey, password); + sshSessionFactory = new SshdSessionFactoryBuilder() + .setPreferredAuthentications("publickey") + .setDefaultKeysProvider(file -> keyPairs) + .setHomeDirectory(directory) + .setSshDirectory(directory) + .setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() { + @Override + public List lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) { + return Collections.emptyList(); + } + + @Override + public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) { + return true; + } + }) + .build(new JGitKeyCache()); + } + return sshSessionFactory; + } + + private static Iterable loadKeyPairs(String privateKeyContent, String password) { + Iterable keyPairs = null; + try { + keyPairs = SecurityUtils.loadKeyPairIdentities(null, + null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password); + } catch (Exception e) {} + if (keyPairs == null) { + throw new IllegalArgumentException("Failed to load ssh private key"); + } + return keyPairs; + } @Data public static class Commit { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 3b639659c8..d88fabc052 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class AdminSettings extends BaseData { +public class AdminSettings extends BaseData implements HasTenantId { private static final long serialVersionUID = -7670322981725511892L; + private TenantId tenantId; + @NoXss @Length(fieldName = "key") private String key; @@ -44,6 +47,7 @@ public class AdminSettings extends BaseData { public AdminSettings(AdminSettings adminSettings) { super(adminSettings); + this.tenantId = adminSettings.getTenantId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -60,7 +64,16 @@ public class AdminSettings extends BaseData { return super.getCreatedTime(); } - @ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") + @ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true) + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + @ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") public String getKey() { return key; } @@ -69,7 +82,7 @@ public class AdminSettings extends BaseData { this.key = key; } - @ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value") + @ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value") public JsonNode getJsonValue() { return jsonValue; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index f1391555b0..52ad469ce1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -17,6 +17,10 @@ package org.thingsboard.server.common.data; public class StringUtils { + public static final String EMPTY = ""; + + public static final int INDEX_NOT_FOUND = -1; + public static boolean isEmpty(String source) { return source == null || source.isEmpty(); } @@ -32,4 +36,42 @@ public class StringUtils { public static boolean isNotBlank(String source) { return source != null && !source.isEmpty() && !source.trim().isEmpty(); } + + public static String removeStart(final String str, final String remove) { + if (isEmpty(str) || isEmpty(remove)) { + return str; + } + if (str.startsWith(remove)){ + return str.substring(remove.length()); + } + return str; + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringBetween(final String str, final String open, final String close) { + if (str == null || open == null || close == null) { + return null; + } + final int start = str.indexOf(open); + if (start != INDEX_NOT_FOUND) { + final int end = str.indexOf(close, start + open.length()); + if (end != INDEX_NOT_FOUND) { + return str.substring(start + open.length(), end); + } + } + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ce4f8e71d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java @@ -13,14 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.vc; import lombok.Data; @Data public class EntitiesVersionControlSettings { private String repositoryUri; + private VersionControlAuthMethod authMethod; private String username; private String password; + private String privateKeyFileName; + private String privateKey; + private String privateKeyPassword; private String defaultBranch; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java new file mode 100644 index 0000000000..9d2dfa048e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.vc; + +public enum VersionControlAuthMethod { + USERNAME_PASSWORD, + PRIVATE_KEY +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 0b74e79f2b..2bb9ef6bdb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -90,6 +90,8 @@ public class ModelConstants { * Cassandra admin_settings constants. */ public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; + + public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 49d5002af2..4da17d2c34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -22,14 +22,18 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; +import java.util.UUID; + import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY; @@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY @Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME) public final class AdminSettingsEntity extends BaseSqlEntity implements BaseEntity { + @Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY) + private UUID tenantId; + @Column(name = ADMIN_SETTINGS_KEY_PROPERTY) private String key; @@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl this.setUuid(adminSettings.getId().getId()); } this.setCreatedTime(adminSettings.getCreatedTime()); + this.tenantId = adminSettings.getTenantId().getId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettings toData() { AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id)); adminSettings.setCreatedTime(createdTime); + adminSettings.setTenantId(TenantId.fromUUID(tenantId)); adminSettings.setKey(key); adminSettings.setJsonValue(jsonValue); return adminSettings; diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index faea2cc34e..14455cddd4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.UUID; + public interface AdminSettingsDao extends Dao { /** @@ -35,6 +37,6 @@ public interface AdminSettingsDao extends Dao { * @param key the key * @return the admin settings object */ - AdminSettings findByKey(TenantId tenantId, String key); + AdminSettings findByTenantIdAndKey(UUID tenantId, String key); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bf4c71d296..0462cc38de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -46,20 +47,35 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) { log.trace("Executing findAdminSettingsByKey [{}]", key); Validator.validateString(key, "Incorrect key " + key); - return adminSettingsDao.findByKey(tenantId, key); + return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key); } @Override public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { + if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } + } else if (adminSettings.getKey().equals("entitiesVersionControl")) { + VersionControlAuthMethod authMethod = VersionControlAuthMethod.valueOf(adminSettings.getJsonValue().get("authMethod").asText()); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && !adminSettings.getJsonValue().has("password")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("password", vcSettings.getJsonValue().get("password").asText()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && !adminSettings.getJsonValue().has("privateKey")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKey", vcSettings.getJsonValue().get("privateKey").asText()); + if (!adminSettings.getJsonValue().has("privateKeyPassword") && vcSettings.getJsonValue().has("privateKeyPassword")) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKeyPassword", vcSettings.getJsonValue().get("privateKeyPassword").asText()); + } + } + } } - return adminSettingsDao.save(tenantId, adminSettings); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 58423645b1..10275a3c1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -25,5 +25,6 @@ import java.util.UUID; */ public interface AdminSettingsRepository extends JpaRepository { - AdminSettingsEntity findByKey(String key); + AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 438274e3cc..4a4f7e5440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -46,7 +46,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDaoorg.eclipse.jgit ${jgit.version} + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + ${jgit.version} + From 2eb94fd95e5eca1973a689802af0279124f61a7d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:38:06 +0300 Subject: [PATCH 094/262] Fix beans cycle dependency --- .../DefaultEntitiesVersionControlService.java | 2 -- .../sync/vc/EntitiesVersionControlService.java | 2 ++ .../sync/vc/LocalGitVersionControlService.java | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 1496882031..70bf5b0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -79,8 +79,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntityService entityService; private final TransactionTemplate transactionTemplate; - public static final String SETTINGS_KEY = "entitiesVersionControl"; - @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var commit = gitService.prepareCommit(user.getTenantId(), request); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 8c18aeb32d..d24fa1411e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -32,6 +32,8 @@ import java.util.List; public interface EntitiesVersionControlService { + String SETTINGS_KEY = "entitiesVersionControl"; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index aa51fc3ea1..94976438e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -38,6 +39,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -63,14 +65,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; - private final EntitiesVersionControlService entitiesVersionControlService; + private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = entitiesVersionControlService.getVersionControlSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { gitRepositoryService.initRepository(tenantId, settings); @@ -213,6 +215,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + private List listVersions(TenantId tenantId, String branch, String path) { try { return gitRepositoryService.listVersions(tenantId, branch, path); From 587066cfd32b17e6b0259f147b32987a68c94a63 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 13:19:43 +0300 Subject: [PATCH 095/262] Ability to remove version control settings --- .../server/controller/AdminController.java | 24 ++++++++++++++++--- .../DefaultEntitiesVersionControlService.java | 7 ++++++ .../vc/EntitiesVersionControlService.java | 2 ++ .../vc/LocalGitVersionControlService.java | 14 +++++++++++ .../dao/settings/AdminSettingsService.java | 2 ++ .../sync/vc/DefaultGitRepositoryService.java | 3 ++- .../service/sync/vc/GitRepositoryService.java | 2 ++ .../sync/vc/GitVersionControlService.java | 2 ++ .../server/dao/settings/AdminSettingsDao.java | 2 ++ .../settings/AdminSettingsServiceImpl.java | 6 +++++ .../sql/settings/AdminSettingsRepository.java | 4 ++++ .../dao/sql/settings/JpaAdminSettingsDao.java | 9 +++++++ 12 files changed, 73 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 237f681819..bf8c656257 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; @@ -38,8 +41,8 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; -import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -203,7 +206,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vsSettings") + @PostMapping("/vcSettings") public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -213,6 +216,21 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + versionControlService.deleteVersionControlSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 70bf5b0d5b..c69ff94630 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -312,6 +312,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return savedVersionControlSettings; } + @Override + public void deleteVersionControlSettings(TenantId tenantId) { + if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + gitService.clearRepository(tenantId); + } + } + @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index d24fa1411e..28f095123c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -58,6 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + void deleteVersionControlSettings(TenantId tenantId); + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index 94976438e9..d73157d256 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -111,6 +111,20 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + @Override + public void clearRepository(TenantId tenantId) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.clearRepository(tenantId); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + @Override public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { var lock = getRepoLock(tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index d9238c9d7b..77f542ccc3 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -27,4 +27,6 @@ public interface AdminSettingsService { AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); + boolean deleteAdminSettings(TenantId tenantId, String key); + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 56ca38ffe4..aac40e0dcb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -224,7 +224,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { repositories.put(tenantId, repository); } - private void clearRepository(TenantId tenantId) throws IOException { + @Override + public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { FileUtils.deleteDirectory(new File(repository.getDirectory())); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 4b1843e3c7..19754750c2 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -36,6 +36,8 @@ public interface GitRepositoryService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void clearRepository(TenantId tenantId) throws IOException; + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index 8345375e4f..f28584c418 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -34,6 +34,8 @@ public interface GitVersionControlService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + void clearRepository(TenantId tenantId); + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); void addToCommit(PendingCommit commit, EntityExportData> entityData); diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index 14455cddd4..d211a8340f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -39,4 +39,6 @@ public interface AdminSettingsDao extends Dao { */ AdminSettings findByTenantIdAndKey(UUID tenantId, String key); + boolean removeByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 0bb5564fa6..433d056186 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -62,4 +62,10 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { return adminSettingsDao.save(tenantId, adminSettings); } + @Override + public boolean deleteAdminSettings(TenantId tenantId, String key) { + log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); + Validator.validateString(key, "Incorrect key " + key); + return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 10275a3c1a..0e68327fdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -27,4 +27,8 @@ public interface AdminSettingsRepository extends JpaRepository Date: Thu, 19 May 2022 13:21:19 +0300 Subject: [PATCH 096/262] Clear repository on settings update --- .../service/sync/vc/DefaultEntitiesVersionControlService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c69ff94630..dcc96b7d17 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -305,6 +305,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException("Failed to load version control settings!", e); } try { + gitService.clearRepository(tenantId); gitService.initRepository(tenantId, savedVersionControlSettings); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); From 10248aa9f64c5d90d2692cf35f7b803b0f11d384 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 19 May 2022 17:49:40 +0300 Subject: [PATCH 097/262] fix of the packge list --- .../server/vc/ThingsboardVersionControlExecutorApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index 2724a566ee..d30818110d 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -25,7 +25,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; From 76af741399fc01f2eb2b735802f00eb2e932f755 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 15:02:30 +0300 Subject: [PATCH 098/262] UI: Implement git settings form --- .../server/controller/AdminController.java | 8 +- .../DefaultEntitiesVersionControlService.java | 21 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 2 + ui-ngx/src/app/core/http/admin.service.ts | 20 ++ ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../version-control-settings.component.html | 110 ++++++++++ .../version-control-settings.component.scss | 33 +++ .../version-control-settings.component.ts | 198 ++++++++++++++++++ ui-ngx/src/app/shared/models/constants.ts | 3 +- .../src/app/shared/models/settings.models.ts | 21 ++ .../assets/locale/locale.constant-en_US.json | 24 ++- 13 files changed, 459 insertions(+), 13 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index bf8c656257..45b8aa5f95 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -207,10 +207,14 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public EntitiesVersionControlSettings saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - versionControlService.saveVersionControlSettings(getTenantId(), settings); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.saveVersionControlSettings(getTenantId(), settings)); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..d46884ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -291,11 +291,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + EntitiesVersionControlSettings storedSettings = null; + if (adminSettings != null) { + try { + storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - AdminSettings adminSettings = new AdminSettings(); - adminSettings.setTenantId(tenantId); - adminSettings.setKey(SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -341,8 +351,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { if (storedSettings != null) { settings.setPrivateKey(storedSettings.getPrivateKey()); - if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && - StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 7257cfb57b..de4a2fd5ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -51,6 +52,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDao { + return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings', versionControlSettings, + defaultHttpOptionsFromConfig(config)); + } + + public deleteEntitiesVersionControlSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + } + + public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + } + public checkUpdates(config?: RequestConfig): Observable { return this.http.get(`/api/admin/updates`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 4a52fc66f6..85cf24e04d 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -350,7 +350,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '80px', + height: '120px', icon: 'settings', pages: [ { @@ -366,6 +366,13 @@ export class MenuService { type: 'link', path: '/settings/resources-library', icon: 'folder' + }, + { + id: guid(), + name: 'admin.git-settings', + type: 'link', + path: '/settings/vc', + icon: 'manage_history' } ] } @@ -500,6 +507,11 @@ export class MenuService { name: 'resource.resources-library', icon: 'folder', path: '/settings/resources-library' + }, + { + name: 'admin.git-settings', + icon: 'manage_history', + path: '/settings/vc', } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index ddccbb4da4..6c4963d06e 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -32,6 +32,7 @@ import { ResourcesLibraryTableConfigResolver } from '@home/pages/admin/resource/ import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -183,6 +184,19 @@ const routes: Routes = [ } } ] + }, + { + path: 'vc', + component: VersionControlSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.git-settings', + breadcrumb: { + label: 'admin.git-settings', + icon: 'manage_history' + } + } } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 326b16f950..cb81e56020 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -28,6 +28,7 @@ import { SmsProviderComponent } from '@home/pages/admin/sms-provider.component'; import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dialog.component'; import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @NgModule({ declarations: @@ -39,7 +40,8 @@ import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources- SecuritySettingsComponent, OAuth2SettingsComponent, HomeSettingsComponent, - ResourcesLibraryComponent + ResourcesLibraryComponent, + VersionControlSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html new file mode 100644 index 0000000000..b512bb96f5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html @@ -0,0 +1,110 @@ + +
+ + +
+ admin.git-repository-settings + +
+
+ + + +
+ +
+
+ + admin.repository-url + + + admin.repository-url-required + + + + admin.default-branch + + +
+ admin.authentication-settings + + admin.auth-method + + + {{versionControlAuthMethodTranslations.get(method) | translate}} + + + +
+ + common.username + + + + {{ 'admin.change-password-access-token' | translate }} + + + admin.password-access-token + + + +
+
+ + + + {{ 'admin.change-passphrase' | translate }} + + + admin.passphrase + + + +
+
+
+ + + + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss new file mode 100644 index 0000000000..ede3570e68 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts new file mode 100644 index 0000000000..c2626ec062 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts @@ -0,0 +1,198 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + EntitiesVersionControlSettings, + VersionControlAuthMethod, + versionControlAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; + +@Component({ + selector: 'tb-version-control-settings', + templateUrl: './version-control-settings.component.html', + styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] +}) +export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + versionControlSettingsForm: FormGroup; + settings: EntitiesVersionControlSettings = null; + + versionControlAuthMethod = VersionControlAuthMethod; + versionControlAuthMethods = Object.values(VersionControlAuthMethod); + versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.versionControlSettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: [null, []], + authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + (settings) => { + this.settings = settings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + }); + } + + checkAccess(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.checkVersionControlAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-git-settings-title', ), + this.translate.instant('admin.delete-git-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteEntitiesVersionControlSettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + } + ); + } + }); + } + + confirmForm(): FormGroup { + return this.versionControlSettingsForm; + } + + changePasswordChanged() { + if (this.changePassword) { + this.versionControlSettingsForm.get('password').patchValue(''); + this.versionControlSettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); + this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; + if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.versionControlSettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.versionControlSettingsForm.get('password').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('password').disable({emitEvent}); + } + this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.versionControlSettingsForm.get('username').disable({emitEvent}); + this.versionControlSettingsForm.get('password').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.versionControlSettingsForm.get('privateKey').clearValidators(); + } else { + this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 4bfce6414a..7c970678b8 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -133,7 +133,8 @@ export const HelpLinks = { widgetsConfigAlarm: helpBaseUrl + '/docs/user-guide/ui/dashboards#alarm', widgetsConfigStatic: helpBaseUrl + '/docs/user-guide/ui/dashboards#static', ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', - ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge' + ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', + versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 615d0994c9..8a1b316001 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -396,3 +396,24 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid } return smsProviderConfiguration; } + +export enum VersionControlAuthMethod { + USERNAME_PASSWORD = 'USERNAME_PASSWORD', + PRIVATE_KEY = 'PRIVATE_KEY' +} + +export const versionControlAuthMethodTranslationMap = new Map([ + [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +]); + +export interface EntitiesVersionControlSettings { + repositoryUri: string; + defaultBranch: string; + authMethod: VersionControlAuthMethod; + username: string; + password: string; + privateKeyFileName: string; + privateKey: string; + privateKeyPassword: string; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cf576aedbf..5aaadb2bd6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -311,8 +311,28 @@ "scheme-music-codes": "10 - Music Codes (ISO-2022-JP)", "scheme-extended-kanji-jis": "13 - Extended Kanji JIS (X 0212-1990)", "scheme-korean-graphic-character-set": "14 - Korean Graphic Character Set (KS C 5601/KS X 1001)" - } - }, + }, + "git-settings": "Git settings", + "git-repository-settings": "Git repository settings", + "repository-url": "Repository URL", + "repository-url-required": "Repository URL is required.", + "default-branch": "Default branch name", + "authentication-settings": "Authentication settings", + "auth-method": "Authentication method", + "auth-method-username-password": "Password / access token", + "auth-method-private-key": "Private key", + "password-access-token": "Password / access token", + "change-password-access-token": "Change password / access token", + "private-key": "Private key", + "drop-private-key-file-or": "Drag and drop a private key file or", + "passphrase": "Passphrase", + "enter-passphrase": "Enter passphrase", + "change-passphrase": "Change passphrase", + "check-access": "Check access", + "check-vc-access-success": "Git repository access successfully verified!", + "delete-git-settings-title": "Are you sure you want to delete git settings?", + "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + }, "alarm": { "alarm": "Alarm", "alarms": "Alarms", From 7071c7a6bc9deb72bf579e4db868c7d98146c529 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 20 May 2022 17:33:24 +0300 Subject: [PATCH 099/262] Implementation of the Git Queue service client --- .../server/controller/BaseController.java | 21 + .../EntitiesVersionControlController.java | 85 ++-- .../install/SqlDatabaseUpgradeService.java | 12 - .../queue/DefaultTbClusterService.java | 8 + .../service/sync/vc/CommitGitRequest.java | 33 ++ .../DefaultEntitiesVersionControlService.java | 215 +++++---- .../DefaultGitVersionControlQueueService.java | 287 +++++++++++ .../sync/vc/EntitiesContentGitRequest.java | 37 ++ .../vc/EntitiesVersionControlService.java | 19 +- .../sync/vc/EntityContentGitRequest.java | 34 ++ .../vc/GitVersionControlQueueService.java | 57 +++ .../sync/vc/ListBranchesGitRequest.java | 29 ++ .../sync/vc/ListEntitiesGitRequest.java | 30 ++ .../sync/vc/ListVersionsGitRequest.java | 31 ++ .../vc/LocalGitVersionControlService.java | 452 +++++++++--------- .../service/sync/vc/PendingGitRequest.java | 37 ++ .../src/main/resources/thingsboard.yml | 5 +- .../server/cluster/TbClusterService.java | 7 +- common/cluster-api/src/main/proto/queue.proto | 96 +++- .../server/common/msg/queue/ServiceType.java | 2 +- .../queue/discovery/HashPartitionService.java | 10 +- .../queue/kafka/TbKafkaTopicConfigs.java | 6 + .../provider/AwsSqsMonolithQueueFactory.java | 7 + .../provider/AwsSqsTbCoreQueueFactory.java | 7 + .../InMemoryMonolithQueueFactory.java | 6 + .../provider/KafkaMonolithQueueFactory.java | 36 +- .../provider/KafkaTbCoreQueueFactory.java | 20 + .../KafkaTbVersionControlQueueFactory.java | 116 +++++ .../provider/PubSubMonolithQueueFactory.java | 7 + .../provider/PubSubTbCoreQueueFactory.java | 7 + .../RabbitMqMonolithQueueFactory.java | 7 + .../provider/RabbitMqTbCoreQueueFactory.java | 7 + .../ServiceBusMonolithQueueFactory.java | 7 + .../ServiceBusTbCoreQueueFactory.java | 7 + .../queue/provider/TbCoreQueueFactory.java | 8 + .../provider/TbCoreQueueProducerProvider.java | 8 + .../provider/TbQueueProducerProvider.java | 8 + .../TbRuleEngineProducerProvider.java | 6 + .../TbTransportQueueProducerProvider.java | 6 + .../TbVersionControlProducerProvider.java | 84 ++++ .../TbVersionControlQueueFactory.java | 44 ++ .../TbQueueVersionControlSettings.java | 34 ++ .../queue/util/TbVersionControlComponent.java | 26 + common/version-control/pom.xml | 24 - .../sync/vc/GitVersionControlService.java | 11 - .../version-control/src/main/proto/vc.proto | 71 --- 46 files changed, 1579 insertions(+), 498 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java delete mode 100644 common/version-control/src/main/proto/vc.proto diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 195ea9a3ee..709b809823 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -18,6 +18,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -27,6 +31,7 @@ import org.springframework.http.MediaType; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -948,4 +953,20 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + protected DeferredResult wrapFuture(ListenableFuture future) { + final DeferredResult deferredResult = new DeferredResult<>(); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(T result) { + deferredResult.setResult(result); + } + + @Override + public void onFailure(Throwable t) { + deferredResult.setErrorResult(t); + } + }, MoreExecutors.directExecutor()); + return deferredResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..4c5c7dad65 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,16 +15,26 @@ */ package org.thingsboard.server.controller; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.claim.ClaimResponse; +import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -34,6 +44,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -90,16 +101,15 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/version") - public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - return versionControlService.saveEntitiesVersion(user, request); + return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } } - @ApiOperation(value = "", notes = "" + "```\n[\n" + " {\n" + @@ -108,12 +118,12 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + public DeferredResult> listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId)); } catch (Exception e) { throw handleException(e); } @@ -127,10 +137,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + public DeferredResult> listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType)); } catch (Exception e) { throw handleException(e); } @@ -152,9 +162,9 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + public DeferredResult> listVersions(@PathVariable String branch) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + return wrapFuture(versionControlService.listVersions(getTenantId(), branch)); } catch (Exception e) { throw handleException(e); } @@ -162,21 +172,21 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping("/entity/{branch}/{entityType}/{versionId}") - public List listEntitiesAtVersion(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); + return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } catch (Exception e) { throw handleException(e); } } @GetMapping("/entity/{branch}/{versionId}") - public List listAllEntitiesAtVersion(@PathVariable String branch, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } catch (Exception e) { throw handleException(e); } @@ -216,20 +226,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - String versionId = request.getVersionId(); - if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); - } else { - throw new IllegalArgumentException("No versions available in branch"); - } - } - - return versionControlService.loadEntitiesVersion(user, request); + return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } @@ -252,19 +252,22 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n\n```") @GetMapping("/branches") - public List listBranches() throws ThingsboardException { + public DeferredResult> listBranches() throws ThingsboardException { try { - List remoteBranches = versionControlService.listBranches(getTenantId()); - List infos = new ArrayList<>(); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); - String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); - infos.add(new BranchInfo(defaultBranch, true)); - } + String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); - return infos; + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + }, MoreExecutors.directExecutor())); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b1bdbf61b8..db2352b706 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -647,18 +647,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.3.4": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); - loadSql(schemaUpdateFile, conn); - log.info("Updating schema settings..."); - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); - log.info("Schema updated."); - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2806c35cda..4a52ea2287 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -135,6 +135,14 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java new file mode 100644 index 0000000000..b510ffa62c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +public class CommitGitRequest extends PendingGitRequest { + + private final VersionCreateRequest request; + + public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { + super(tenantId); + this.request = request; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..3fb47a8bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,12 +15,18 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,7 +39,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.*; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -56,6 +61,8 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -72,118 +79,137 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final GitVersionControlService gitService; + private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; + private ListeningExecutorService executor; + + @Value("${vc.thread_pool_size:4}") + private int threadPoolSize; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class)); + } + + @PreDestroy + public void shutdown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + @SuppressWarnings("UnstableApiUsage") @Override - public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); + public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - switch (request.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); - break; - } - case COMPLEX: { - ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitService.deleteAll(commit, entityType); - } + return Futures.transformAsync(pendingCommit, commit -> { + List> gitFutures = new ArrayList<>(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + break; + } + case COMPLEX: { + ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { + gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + } - if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - try { - saveEntityData(user, commit, entity.getId(), config); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } else { - for (UUID entityId : config.getEntityIds()) { - try { - saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } } } - } - - }); - break; + }); + break; + } } - } - - return gitService.push(commit); + return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + }, executor); } - private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { + private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - gitService.addToCommit(commit, entityData); + return gitServiceQueue.addToCommit(commit, entityData); } - @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, externalId); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, entityType); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch); } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override - public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId); + public ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId); } + @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); + ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + Futures.transform(future, entityData -> { + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + }, executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return transactionTemplate.execute(status -> { + return executor.submit(() -> transactionTemplate.execute(status -> { Map results = new HashMap<>(); Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); @@ -199,9 +225,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { int limit = 100; int offset = 0; - List> entityDataList; + List entityDataList; do { - entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) @@ -215,7 +241,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } offset += limit; importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); } while (entityDataList.size() == limit); } catch (Exception e) { throw new RuntimeException(e); @@ -263,7 +289,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } return new ArrayList<>(results.values()); - }); + })); } default: throw new IllegalArgumentException("Unsupported version load request"); @@ -272,8 +298,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listBranches(TenantId tenantId) throws Exception { - return gitService.listBranches(tenantId); + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + return gitServiceQueue.listBranches(tenantId); } @Override @@ -304,32 +330,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } + //TODO: ashvayka +// try { +// gitService.clearRepository(tenantId); +// gitService.initRepository(tenantId, savedVersionControlSettings); +// } catch (Exception e) { +// throw new RuntimeException("Failed to init repository!", e); +// } return savedVersionControlSettings; } @Override public void deleteVersionControlSettings(TenantId tenantId) { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { - gitService.clearRepository(tenantId); - } + //TODO: ashvayka +// if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { +// gitService.clearRepository(tenantId); +// } } @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); settings = this.restoreCredentials(settings, storedSettings); - try { - gitService.testRepository(tenantId, settings); - } catch (Exception e) { - throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), - ThingsboardErrorCode.GENERAL); - } + //TODO: ashvayka +// try { +// gitService.testRepository(tenantId, settings); +// } catch (Exception e) { +// throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), +// ThingsboardErrorCode.GENERAL); +// } } private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java new file mode 100644 index 0000000000..410847fb6b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -0,0 +1,287 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +@TbCoreComponent +@Service +@RequiredArgsConstructor +public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final TbServiceInfoProvider serviceInfoProvider; + private final TbClusterService clusterService; + private final Map> pendingRequestMap = new HashMap<>(); + + @Override + public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + SettableFuture future = SettableFuture.create(); + + CommitGitRequest commit = new CommitGitRequest(tenantId, request); + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPrepareMsg(getCommitPrepareMsg(request)).build() + ).build(), wrap(future, commit)); + + return future; + } + + @Override + public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJson(entityDataJson).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + return future; + } + + @Override + public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityType, null); + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setDeleteMsg( + TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + + return future; + } + + @Override + public ListenableFuture push(CommitGitRequest commit) { + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPushMsg( + TransportProtos.PushMsg.newBuilder().build() + ).build() + ).build(), wrap(commit.getFuture())); + + return commit.getFuture(); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .build()); + } + + private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { + ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .build()); + } + + private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { + ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listBranches(TenantId tenantId) { + ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + + @Override + public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { + EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() + , wrap(request.getFuture())); + + return request.getFuture(); +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } + } + + @Override + public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { + EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); + + registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityType.name()) + .setOffset(offset) + .setLimit(limit) + ).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + private static TbQueueCallback wrap(SettableFuture future) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static TbQueueCallback wrap(SettableFuture future, T value) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + future.set(value); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static String getRelativePath(EntityType entityType, EntityId entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + } + + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request) { + var tenantId = request.getTenantId(); + var requestId = request.getRequestId(); + return ToVersionControlServiceMsg.newBuilder() + .setNodeId(serviceInfoProvider.getServiceId()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setRequestIdMSB(requestId.getMostSignificantBits()) + .setRequestIdLSB(requestId.getLeastSignificantBits()); + + } +} + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java new file mode 100644 index 0000000000..2ded932e44 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +import java.util.List; + +@Getter +public class EntitiesContentGitRequest extends PendingGitRequest> { + + private final String versionId; + private final EntityType entityType; + + public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { + super(tenantId); + this.versionId = versionId; + this.entityType = entityType; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..02afafedad 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -34,25 +35,23 @@ public interface EntitiesVersionControlService { String SETTINGS_KEY = "entitiesVersionControl"; - VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; + ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; - List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - - - List listBranches(TenantId tenantId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java new file mode 100644 index 0000000000..c9ce3c76af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Getter +public class EntityContentGitRequest extends PendingGitRequest { + + private final String versionId; + private final EntityId entityId; + + public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { + super(tenantId); + this.versionId = versionId; + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java new file mode 100644 index 0000000000..e7b7171ea0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlQueueService { + + ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + + ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); + + ListenableFuture deleteAll(CommitGitRequest pendingCommit, EntityType entityType); + + ListenableFuture push(CommitGitRequest commit); + + ListenableFuture> listVersions(TenantId tenantId, String branch); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + ListenableFuture> listBranches(TenantId tenantId); + + ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); + + ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java new file mode 100644 index 0000000000..c8219641da --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListBranchesGitRequest extends PendingGitRequest> { + + public ListBranchesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java new file mode 100644 index 0000000000..0c7216ef49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListEntitiesGitRequest extends PendingGitRequest> { + + public ListEntitiesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java new file mode 100644 index 0000000000..0c8e51042b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public class ListVersionsGitRequest extends PendingGitRequest> { + + public ListVersionsGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..66803e062d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -19,18 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +35,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -60,235 +56,235 @@ import java.util.stream.Collectors; @RequiredArgsConstructor @Service @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService implements GitVersionControlService { +public class LocalGitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); - if (settings != null) { - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - } - - @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.testRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void clearRepository(TenantId tenantId) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.clearRepository(tenantId); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - var pendingCommit = new PendingCommit(tenantId, request); - PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); - if (old != null) { - gitRepositoryService.abort(old); - } - gitRepositoryService.prepareCommit(pendingCommit); - return pendingCommit; - } finally { - lock.unlock(); - } - } - - @Override - public void deleteAll(PendingCommit commit, EntityType entityType) { - doInsideLock(commit, c -> { - try { - gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public void addToCommit(PendingCommit commit, EntityExportData> entityData) { - doInsideLock(commit, c -> { - String entityDataJson; - try { - entityDataJson = jsonWriter.writeValueAsString(entityData); - gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString()), entityDataJson); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); - } - - @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { - try { - return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { - return listEntitiesAtVersion(tenantId, branch, versionId, null); - } - - @Override - public List listBranches(TenantId tenantId) { - return gitRepositoryService.listBranches(tenantId); - } - - @Override - public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { - return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() - .skip(offset).limit(limit) - .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) - .collect(Collectors.toList()); - } - - @Override - public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { - try { - String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, - getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - } - - private List listVersions(TenantId tenantId, String branch, String path) { - try { - return gitRepositoryService.listVersions(tenantId, branch, path); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private void doInsideLock(PendingCommit commit, Consumer r) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - r.accept(commit); - } finally { - lock.unlock(); - } - } - - private T executeInsideLock(PendingCommit commit, Function c) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - return c.apply(commit); - } finally { - lock.unlock(); - } - } - - private void checkCommit(PendingCommit commit) { - PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); - if (existing == null || !existing.getTxId().equals(commit.getTxId())) { - throw new ConcurrentModificationException(); - } - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; - } - - private Lock getRepoLock(TenantId tenantId) { - return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); - } +// private final Map pendingCommitMap = new HashMap<>(); +// +// @AfterStartUp +// public void init() { +// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { +// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); +// if (settings != null) { +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// log.warn("Failed to init repository for tenant {}", tenantId, e); +// } +// } +// }); +// } +// +// @Override +// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.testRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void clearRepository(TenantId tenantId) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.clearRepository(tenantId); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// var pendingCommit = new PendingCommit(tenantId, request); +// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); +// if (old != null) { +// gitRepositoryService.abort(old); +// } +// gitRepositoryService.prepareCommit(pendingCommit); +// return pendingCommit; +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void deleteAll(PendingCommit commit, EntityType entityType) { +// doInsideLock(commit, c -> { +// try { +// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { +// doInsideLock(commit, c -> { +// String entityDataJson; +// try { +// entityDataJson = jsonWriter.writeValueAsString(entityData); +// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), +// entityData.getEntity().getId().toString()), entityDataJson); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public VersionCreationResult push(PendingCommit commit) { +// return executeInsideLock(commit, gitRepositoryService::push); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch) { +// return listVersions(tenantId, branch, (String) null); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { +// return listVersions(tenantId, branch, getRelativePath(entityType, null)); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { +// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { +// try { +// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { +// return listEntitiesAtVersion(tenantId, branch, versionId, null); +// } +// +// @Override +// public List listBranches(TenantId tenantId) { +// return gitRepositoryService.listBranches(tenantId); +// } +// +// @Override +// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { +// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() +// .skip(offset).limit(limit) +// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) +// .collect(Collectors.toList()); +// } +// +// @Override +// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { +// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); +// if (adminSettings != null) { +// try { +// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); +// } catch (Exception e) { +// throw new RuntimeException("Failed to load version control settings!", e); +// } +// } +// return null; +// } +// +// private List listVersions(TenantId tenantId, String branch, String path) { +// try { +// return gitRepositoryService.listVersions(tenantId, branch, path); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private void doInsideLock(PendingCommit commit, Consumer r) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// r.accept(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private T executeInsideLock(PendingCommit commit, Function c) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// return c.apply(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private void checkCommit(PendingCommit commit) { +// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); +// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { +// throw new ConcurrentModificationException(); +// } +// } +// +// private String getRelativePath(EntityType entityType, String entityId) { +// String path = entityType.name().toLowerCase(); +// if (entityId != null) { +// path += "/" + entityId + ".json"; +// } +// return path; +// } +// +// private Lock getRepoLock(TenantId tenantId) { +// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java new file mode 100644 index 0000000000..3650f9e113 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.SettableFuture; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Getter +public class PendingGitRequest { + + private final UUID requestId; + private final TenantId tenantId; + private final SettableFuture future; + + public PendingGitRequest(TenantId tenantId) { + this.requestId = UUID.randomUUID(); + this.tenantId = tenantId; + this.future = SettableFuture.create(); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 650f110dd7..1032c79054 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1021,6 +1021,9 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1115,8 +1118,8 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: + thread_pool_size: "${TB_VC_POOL_SIZE:4}" git: - service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" management: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 22953f259a..e2486f3a7e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -31,8 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueClusterService; @@ -47,9 +48,11 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 50f114856f..cb1a1ab4db 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -618,8 +618,8 @@ message TbSubscriptionUpdateValueListProto { } message TbSubscriptionUpdateTsValue { - int64 ts = 1; - optional string value = 2; + int64 ts = 1; + optional string value = 2; } /** @@ -676,6 +676,95 @@ message EdgeNotificationMsgProto { PostAttributeMsg postAttributesMsg = 12; } +/** + TB Core to Version Control Service + */ +message CommitRequestMsg { + PrepareMsg prepareMsg = 1; + AddMsg addMsg = 2; + DeleteMsg deleteMsg = 3; + PushMsg pushMsg = 4; + AbortMsg abortMsg = 5; +} + +message CommitResponseMsg { + string commitId = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string commitMsg = 1; + string branchName = 2; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} + +message ListVersionsRequestMsg { + string branchName = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message ListEntitiesRequestMsg { + string branchName = 1; + string versionId = 2; + string entityType = 3; +} + +message ListBranchesRequestMsg { +} + +message EntityContentRequestMsg { + string versionId = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message EntitiesContentRequestMsg { + string versionId = 1; + string entityType = 2; + int32 offset = 3; + int32 limit = 4; +} + +message ToVersionControlServiceMsg { + string nodeId = 1; + int64 tenantIdMSB = 2; + int64 tenantIdLSB = 3; + int64 requestIdMSB = 4; + int64 requestIdLSB = 5; + CommitRequestMsg commitRequest = 6; + ListVersionsRequestMsg listVersionRequest = 7; + ListEntitiesRequestMsg listEntitiesRequest = 8; + ListBranchesRequestMsg listBranchesRequest = 9; + EntityContentRequestMsg entityContentRequest = 10; + EntitiesContentRequestMsg entitiesContentRequest = 11; +} + +message VersionControlResponseMsg { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + CommitResponseMsg commitResponse = 3; +} + /** * Main messages; */ @@ -730,6 +819,7 @@ message ToCoreNotificationMsg { bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; + VersionControlResponseMsg vcResponseMsg = 7; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -793,3 +883,5 @@ message ToOtaPackageStateServiceMsg { int64 otaPackageIdLSB = 7; string type = 8; } + + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 1c05ac7f66..7f276e7e3f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR, TB_VC_EXECUTOR; public static ServiceType of(String serviceType) { return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index ccf8b9dd92..e2325be878 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,6 +54,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; + @Value("${queue.vc.topic}") + private String vcTopic; + @Value("${queue.vc.partitions:10}") + private Integer vcPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -97,6 +101,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); + partitionSizesMap.put(vcKey, vcPartitions); + partitionTopicsMap.put(vcKey, vcTopic); + List queueRoutingInfoList; String serviceType = serviceInfoProvider.getServiceType(); @@ -401,7 +409,7 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); - } else if (ServiceType.TB_CORE.equals(serviceType)) { + } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 4c9194b675..75d0bd9b2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -40,6 +40,9 @@ public class TbKafkaTopicConfigs { private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; + @Value("${queue.kafka.topic-properties.version-control:}") + private String vcProperties; + @Getter private Map coreConfigs; @@ -53,6 +56,8 @@ public class TbKafkaTopicConfigs { private Map jsExecutorConfigs; @Getter private Map fwUpdatesConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -62,6 +67,7 @@ public class TbKafkaTopicConfigs { notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); fwUpdatesConfigs = getConfigs(fwUpdatesProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 660173c71d..d02b58aa2b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 81736c2b55..89b45e2822 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fe87fc2886..98c1a8029e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -146,6 +146,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 104c531e12..b24ae700bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,6 +52,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbKafkaSettings kafkaSettings; @@ -68,6 +70,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -76,6 +79,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -85,6 +90,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.notificationsTopicService = notificationsTopicService; @@ -95,6 +101,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -103,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -155,6 +163,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("monolith-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { String queueName = configuration.getName(); @@ -311,6 +332,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -331,5 +362,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 58dec0792a..f476a068d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -50,6 +51,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -65,6 +67,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; @@ -74,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; public KafkaTbCoreQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -82,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { @@ -92,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; @@ -101,6 +107,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -282,6 +289,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -302,5 +319,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..598f86d0f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") +public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + this.consumerStatsService = consumerStatsService; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index aceb9d4f0d..a57e30b655 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -207,6 +208,12 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 18a6668581..6cf9aa9a45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 2a724289bb..d4839110e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e728be6085..1dca3ba551 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -169,6 +170,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 88bb0a4045..f7db6379c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -204,6 +205,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e1eb41b2b2..5b471b2830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 4debe1e280..aadefa0292 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateSer import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -122,4 +123,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueProducer> createTransportApiResponseProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + /** + * Used to push messages to instances of TB Version Control Service + * + * @return + */ + TbQueueProducer> createVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index ef0b0c38e9..ed7de35274 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -22,6 +22,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -39,6 +40,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toVersionControl; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -52,6 +54,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); } @Override @@ -83,4 +86,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + return toVersionControl; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 19ebb4666e..19046c5a2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -70,4 +71,11 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbUsageStatsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 26f9df069e..c21ae99b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -83,4 +84,9 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3132ed684f..f9cc139559 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -73,6 +74,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java new file mode 100644 index 0000000000..c8ffc1da0d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") +public class TbVersionControlProducerProvider implements TbQueueProducerProvider { + + private final TbVersionControlQueueFactory tbQueueProvider; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; + + public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java new file mode 100644 index 0000000000..70b16afc68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages from TB Core Service + * + * @return + */ + TbQueueConsumer> createToVersionControlMsgConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java new file mode 100644 index 0000000000..e7f0ce2259 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueVersionControlSettings { + + @Value("${queue.vc.topic:tb_version_control}") + private String topic; + + @Value("${queue.vc.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + + @Value("${queue.vc.partitions:10}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java new file mode 100644 index 0000000000..e72fe60b6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") +public @interface TbVersionControlComponent { +} diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index a8d775bcdc..7b46f7ba72 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,21 +86,6 @@ org.eclipse.jgit org.eclipse.jgit - - io.grpc - grpc-netty-shaded - provided - - - io.grpc - grpc-protobuf - provided - - - io.grpc - grpc-stub - provided - org.eclipse.jgit org.eclipse.jgit.ssh.apache @@ -126,15 +111,6 @@ - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - thingsboard-repo-deploy diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..56349cf8a5 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -16,15 +16,12 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; @@ -36,14 +33,6 @@ public interface GitVersionControlService { void clearRepository(TenantId tenantId); - PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); - - void addToCommit(PendingCommit commit, EntityExportData> entityData); - - void deleteAll(PendingCommit pendingCommit, EntityType entityType); - - VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); List listVersions(TenantId tenantId, String branch, EntityType entityType); diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto deleted file mode 100644 index e8c7e09384..0000000000 --- a/common/version-control/src/main/proto/vc.proto +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "org.thingsboard.server.gen.vc.v1"; -option java_multiple_files = true; -option java_outer_classname = "EdgeProtos"; - -package vc; - -// Interface exported by the ThingsBoard Core. -service TbGitRpcService { - - rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} - -} - - -/** - * Data Structures; - */ -message CommitRequestMsg { - string txId = 1; - PrepareMsg prepareMsg = 2; - AddMsg addMsg = 3; - DeleteMsg deleteMsg = 4; - PushMsg pushMsg = 5; - AbortMsg abortMsg = 6; -} - -message CommitResponseMsg { - string id = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; -} - -message PrepareMsg { - string tenantId = 1; - string commitMsg = 2; - string branchName = 3; -} - -message AddMsg { - string relativePath = 1; - string entityDataJson = 2; -} - -message DeleteMsg { - string relativePath = 1; -} - -message PushMsg { -} - -message AbortMsg { -} \ No newline at end of file From 841f9d5ff6287af4f6ba3e39b7bff3d09a76ab70 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 17:50:35 +0300 Subject: [PATCH 100/262] UI: Single entity Git export --- .../http/entities-version-control.service.ts | 40 ++++ .../home/components/home-components.module.ts | 7 +- .../profile/device-profile.component.html | 6 + .../vc/vc-entity-export-dialog.component.html | 77 +++++++ .../vc/vc-entity-export-dialog.component.ts | 105 +++++++++ .../home/dialogs/home-dialogs.service.ts | 17 ++ .../home/pages/asset/asset.component.html | 6 + .../asset/assets-table-config.resolver.ts | 10 + .../pages/customer/customer.component.html | 6 + .../customers-table-config.resolver.ts | 12 ++ .../dashboard/dashboard-form.component.html | 6 + .../dashboards-table-config.resolver.ts | 12 ++ .../device-profiles-table-config.resolver.ts | 12 ++ .../home/pages/device/device.component.html | 6 + .../device/devices-table-config.resolver.ts | 10 + .../pages/rulechain/rulechain.component.html | 6 + .../rulechains-table-config.resolver.ts | 12 ++ .../vc/branch-autocomplete.component.html | 43 ++++ .../vc/branch-autocomplete.component.ts | 199 ++++++++++++++++++ ui-ngx/src/app/shared/models/vc.models.ts | 55 +++++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 12 ++ 22 files changed, 662 insertions(+), 4 deletions(-) create mode 100644 ui-ngx/src/app/core/http/entities-version-control.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/vc.models.ts diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts new file mode 100644 index 0000000000..231581a02f --- /dev/null +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -0,0 +1,40 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable } from 'rxjs'; +import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; + +@Injectable({ + providedIn: 'root' +}) +export class EntitiesVersionControlService { + + constructor( + private http: HttpClient + ) { + } + + public listBranches(config?: RequestConfig): Observable> { + return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 4a721647ef..a88f0349eb 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -151,6 +151,7 @@ import { DashboardStateComponent } from '@home/components/dashboard-page/dashboa import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; +import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; @NgModule({ declarations: @@ -272,7 +273,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], imports: [ CommonModule, @@ -388,7 +390,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html index 565eb5b160..7e35dbdaf4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html @@ -34,6 +34,12 @@ [fxShow]="!isEdit && !entity?.default"> {{'device-profile.set-default' | translate }} + + + + +
+
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+
+
+
+
+ + +
+
+ +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts new file mode 100644 index 0000000000..9d3affea42 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts @@ -0,0 +1,105 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; +import { ErrorStateMatcher } from '@angular/material/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@app/shared/components/dialog.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +export interface VcEntityExportDialogData { + entityId: EntityId; +} + +@Component({ + selector: 'tb-vc-entity-export-dialog', + templateUrl: './vc-entity-export-dialog.component.html', + providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], + styleUrls: [] +}) +export class VcEntityExportDialogComponent extends DialogComponent + implements OnInit, ErrorStateMatcher { + + exportFormGroup: FormGroup; + + submitted = false; + + createResult: VersionCreationResult; + + createResultMessage: SafeHtml; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, + @SkipSelf() private errorStateMatcher: ErrorStateMatcher, + public dialogRef: MatDialogRef, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private domSanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store, router, dialogRef); + + this.exportFormGroup = this.fb.group({ + branch: [null, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + ngOnInit(): void { + } + + isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { + const originalErrorState = this.errorStateMatcher.isErrorState(control, form); + const customErrorState = !!(control && control.invalid && this.submitted); + return originalErrorState || customErrorState; + } + + cancel(): void { + this.dialogRef.close(); + } + + export(): void { + this.submitted = true; + const request: SingleEntityVersionCreateRequest = { + entityId: this.data.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + this.createResult = result; + const message = this.translate.instant('version-control.export-entity-version-result-message', + {name: result.version.name, commitId: result.version.id}); + this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); + }); + } +} diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 6dc8746b2e..1730c69084 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,6 +22,11 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + VcEntityExportDialogComponent, + VcEntityExportDialogData +} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -41,6 +46,17 @@ export class HomeDialogsService { } } + public exportVcEntity(entityId: EntityId): Observable { + return this.dialog.open(VcEntityExportDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + entityId + } + }).afterClosed(); + } + private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { @@ -53,4 +69,5 @@ export class HomeDialogsService { } }).afterClosed(); } + } diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 00ed882cd9..75171b4532 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,6 +46,12 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} + + + + + + + + + + + + {{ 'version-control.branch-required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts new file mode 100644 index 0000000000..421710181a --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -0,0 +1,199 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable, of } from 'rxjs'; +import { + catchError, + debounceTime, + distinctUntilChanged, + map, + publishReplay, + refCount, + switchMap, + tap +} from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { BranchInfo } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; + +@Component({ + selector: 'tb-branch-autocomplete', + templateUrl: './branch-autocomplete.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => BranchAutocompleteComponent), + multi: true + }] +}) +export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { + + branchFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + @Input() + selectDefaultBranch = true; + + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; + + filteredBranches: Observable>; + + branches: Observable>; + + searchText = ''; + + private dirty = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + this.branchFormGroup = this.fb.group({ + branch: [null, []] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + + this.branches = null; + this.filteredBranches = this.branchFormGroup.get('branch').valueChanges + .pipe( + debounceTime(150), + distinctUntilChanged(), + tap(value => { + this.updateView(value); + }), + map(value => value ? value : ''), + switchMap(branch => this.fetchBranches(branch)) + ); + } + + ngAfterViewInit(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } + + selectDefaultBranchIfNeeded(): void { + if (this.selectDefaultBranch && !this.modelValue) { + this.getBranches().subscribe( + (data) => { + if (data && data.length) { + const defaultBranch = data.find(branch => branch.default); + if (defaultBranch) { + this.modelValue = defaultBranch.name; + this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); + this.propagateChange(this.modelValue); + } + } + } + ); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + if (value != null) { + this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + } else { + this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.selectDefaultBranchIfNeeded(); + } + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.branchFormGroup.get('branch').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + updateView(value: string | null) { + if (this.modelValue !== value) { + this.modelValue = value; + this.propagateChange(this.modelValue); + } + } + + displayBranchFn(branch?: string): string | undefined { + return branch ? branch : undefined; + } + + fetchBranches(searchText?: string): Observable> { + this.searchText = searchText; + return this.getBranches().pipe( + map(branches => branches.map(branch => branch.name).filter(branchName => { + return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; + })) + ); + } + + getBranches(): Observable> { + if (!this.branches) { + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + this.branches = branchesObservable.pipe( + catchError(() => of([] as Array)), + publishReplay(1), + refCount() + ); + } + return this.branches; + } + + clear() { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); + setTimeout(() => { + this.branchInput.nativeElement.blur(); + this.branchInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts new file mode 100644 index 0000000000..0c890ff9b4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -0,0 +1,55 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from '@shared/models/id/entity-id'; + +export interface VersionCreateConfig { + saveRelations: boolean; +} + +export enum VersionCreateRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + COMPLEX = 'COMPLEX' +} + +export interface VersionCreateRequest { + versionName: string; + branch: string; + type: VersionCreateRequestType; +} + +export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { + entityId: EntityId; + config: VersionCreateConfig; + type: VersionCreateRequestType.SINGLE_ENTITY; +} + +export interface BranchInfo { + name: string; + default: boolean; +} + +export interface EntityVersion { + id: string; + name: string; +} + +export interface VersionCreationResult { + version: EntityVersion; + added: number; + modified: number; + removed: number; +} diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 8d8de8e74a..7e73452b18 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -163,6 +163,7 @@ import { HtmlComponent } from '@shared/components/html.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { MultipleImageInputComponent } from '@shared/components/multiple-image-input.component'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -284,7 +285,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ], imports: [ CommonModule, @@ -484,7 +486,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5aaadb2bd6..00880644e4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3049,6 +3049,18 @@ "json-value-invalid": "JSON value has an invalid format", "json-value-required": "JSON value is required." }, + "version-control": { + "branch": "Branch", + "select-branch": "Select branch", + "branch-required": "Branch is required", + "export-entity-version": "Export entity version", + "entity-version-exported": "Entity version successfully exported", + "version-name": "Version name", + "version-name-required": "Version name is required", + "export-entity-relations": "Export entity relations", + "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", + "export-to-git": "Export to Git" + }, "widget": { "widget-library": "Widgets Library", "widget-bundle": "Widgets Bundle", From e114f4d67dc7c81e4fd15fee606da8db999540ee Mon Sep 17 00:00:00 2001 From: blackstar-baba <535650957@qq.com> Date: Mon, 23 May 2022 11:05:36 +0800 Subject: [PATCH 101/262] add chinese translations for edge --- .../assets/locale/locale.constant-zh_CN.json | 197 +++++++++++++++++- 1 file changed, 195 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json index 11e34c9102..6d8aa21ca1 100644 --- a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json +++ b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json @@ -312,6 +312,11 @@ "filter-type-device-type": "设备类型", "filter-type-device-type-and-name-description": "类型为 '{{deviceType}}' 且以 '{{prefix}}' 开头的设备", "filter-type-device-type-description": "类型为 '{{deviceType}}' 的设备", + "filter-type-edge-type": "边缘类型", + "filter-type-edge-type-description": "类型为 '{{edgeType}}' 的边缘", + "filter-type-edge-type-and-name-description": "类型为 '{{edgeType}}' 且以 '{{prefix}}' 开头的边缘", + "filter-type-edge-search-query": "边缘搜索查询", + "filter-type-edge-search-query-description": "类型为 {{edgeTypes}} 且具有 {{relationType}} 关联 {{direction}} {{rootEntity}} 的边缘", "filter-type-entity-list": "实体列表", "filter-type-entity-name": "实体名称", "filter-type-entity-view-search-query": "实体视图搜索查询", @@ -409,6 +414,8 @@ "assets": "资产", "assign-asset-to-customer": "将资产分配给客户", "assign-asset-to-customer-text": "请选择要分配给客户的资产", + "assign-asset-to-edge-text": "请选择要分配给边缘的资产", + "assign-asset-to-edge-title": "将资产分配给边缘", "assign-assets": "分配资产", "assign-assets-text": "分配 { count, plural, 1 {# 个资产} other {# 个资产} } 给客户", "assign-new-asset": "分配新资产", @@ -451,6 +458,9 @@ "type": "类型", "type-required": "类型必填。", "unassign-asset": "未分配资产", + "unassign-asset-from-edge": "取消分配边缘", + "unassign-asset-from-edge-text": "确认后，所有选定的资产将被分配，边缘无法访问。", + "unassign-asset-from-edge-title": "您确定要取消对'{{assetName}}'资产的分配吗？", "unassign-asset-text": "确认后，资产将未分配，客户无法访问。", "unassign-asset-title": "您确定要取消对'{{assetName}}'资产的分配吗？", "unassign-assets": "取消分配资产", @@ -458,6 +468,9 @@ "unassign-assets-text": "确认后，所有选定的资产将被分配，客户无法访问。", "unassign-assets-title": "您确定要取消分配 { count, plural, 1 {# 个资产} other {# 个资产} }吗?", "unassign-from-customer": "取消分配客户", + "unassign-assets-from-edge": "取消分配边缘", + "unassign-assets-from-edge-text": "确认后，所有选定的资产将被分配，边缘无法访问。", + "unassign-assets-from-edge-title": "您确定要取消分配 { count, plural, 1 {1 资产} other {# 个资产} }吗?", "view-assets": "查看资产" }, "attribute": { @@ -512,6 +525,7 @@ "type-alarm-clear": "已清除", "type-assigned-from-tenant": "从租户分配", "type-assigned-to-customer": "分配给客户", + "type-assigned-to-edge": "分配给边缘", "type-assigned-to-tenant": "分配给租户", "type-attributes-deleted": "删除属性", "type-attributes-read": "读取属性", @@ -532,6 +546,7 @@ "type-timeseries-deleted": "遥测数据已删除", "type-timeseries-updated": "遥测数据已更新", "type-unassigned-from-customer": "未分配给客户", + "type-unassigned-from-edge": "未分配给边缘", "type-updated": "更新", "user": "用户" }, @@ -603,6 +618,7 @@ "description": "说明", "details": "详情", "devices": "客户设备", + "edges": "客户边缘实例", "entity-views": "客户实体视图", "events": "事件", "idCopiedMessage": "客户ID已复制到粘贴板", @@ -610,12 +626,15 @@ "manage-customer-assets": "管理客户资产", "manage-customer-dashboards": "管理客户仪表板", "manage-customer-devices": "管理客户设备", + "manage-customer-edges": "管理客户边缘", "manage-customer-users": "管理客户用户", "manage-dashboards": "管理仪表板", "manage-devices": "管理设备", + "manage-edges": "管理边缘", "manage-public-assets": "管理公共资产", "manage-public-dashboards": "管理公共仪表板", "manage-public-devices": "管理公共设备", + "manage-public-edges": "管理公共边缘", "manage-users": "管理用户", "management": "客户管理", "no-customers-matching": "没有找到匹配 '{{entity}}' 的客户。", @@ -624,6 +643,7 @@ "public-dashboards": "公共仪表板", "public-devices": "公共设备", "public-entity-views": "公共实体视图", + "public-edges": "公共边缘", "search": "查找客户", "select-customer": "选择客户", "select-default-customer": "选择默认的客户", @@ -639,6 +659,9 @@ "alias-resolution-error-title": "仪表板别名配置错误", "assign-dashboard-to-customer": "将仪表板分配给客户", "assign-dashboard-to-customer-text": "请选择要分配给客户的仪表板", + "assign-dashboard-to-edge": "将仪表板分配给边缘", + "assign-dashboard-to-edge-text": "请选择要分配给边缘的仪表板", + "assign-dashboard-to-edge-title": "将仪表板分配给边缘", "assign-dashboards": "分配仪表板", "assign-dashboards-text": "分配 { count, plural, 1 {# 个仪表板} other {# 个仪表板} } 给客户", "assign-new-dashboard": "分配新的仪表板", @@ -773,6 +796,7 @@ "title-required": "标题必填。", "toolbar-always-open": "工具栏常驻", "unassign-dashboard": "取消分配仪表板", + "unassign-dashboard-from-edge-text": "确认后，所有选定的仪表板将被取消分配，边缘将无法访问。", "unassign-dashboard-text": "确认后，面板将被取消分配，客户将无法访问。", "unassign-dashboard-title": "您确定要取消分配仪表板 '{{dashboardTitle}}'吗？", "unassign-dashboards": "取消分配仪表板", @@ -780,6 +804,8 @@ "unassign-dashboards-action-title": "取消分配此客户 { count, plural, 1 {# 个仪表板} other {# 个仪表板} }", "unassign-dashboards-text": "确认后，所有选定的仪表板将被取消分配，客户将无法访问。", "unassign-dashboards-title": "确定要取消分配仪表板 { count, plural, 1 {# 个仪表板} other {# 个仪表板} } 吗？", + "unassign-dashboards-from-edge-text": "确认后，所有选定的仪表板将被取消分配，边缘将无法访问。", + "unassign-dashboards-from-edge-title": "确定要取消分配仪表板 { count, plural, 1 {# 个仪表板} other {# 个仪表板} } 吗？", "unassign-from-customer": "取消分配客户", "unassign-from-customers": "客户未分配仪表板", "unassign-from-customers-text": "请选择从仪表板中取消分配的客户", @@ -1021,6 +1047,8 @@ "any-device": "任意设备", "assign-device-to-customer": "将设备分配给客户", "assign-device-to-customer-text": "请选择要分配给客户的设备", + "assign-device-to-edge-text":"请选择要分配给边缘的设备", + "assign-device-to-edge-title": "将设备分配给边缘", "assign-devices": "分配设备", "assign-devices-text": "将 {count,plural,1 {# 个设备} other {# 个设备} }分配给客户", "assign-new-device": "分配新设备", @@ -1106,8 +1134,13 @@ "unassign-device": "取消分配设备", "unassign-device-text": "确认后,设备将被取消分配,客户将无法访问。", "unassign-device-title": "您确定要取消分配设备 '{{deviceName}}'？", + "unassign-device-from-edge-text": "确认后,设备将被取消分配,边缘将无法访问。", + "unassign-device-from-edge-title": "您确定要取消分配设备 '{{deviceName}}'？", "unassign-devices": "取消分配设备", "unassign-devices-action-title": "取消分配此客户 {count,plural,1 {# 个设备} other {# 个设备} }", + "unassign-devices-from-edge": "取消分配边缘", + "unassign-devices-from-edge-text": "确认后,设备将被取消分配，边缘将无法访问。", + "unassign-devices-from-edge-title": "您确定要取消分配 { count, plural, 1 {1个 设备} other {# 个设备} } 吗?", "unassign-devices-text": "确认后,所有选定的设备将被取消分配,并且客户将无法访问。", "unassign-devices-title": "确定要取消分配 {count,plural,1 {# 个设备} other {# 个设备} } 吗？", "unassign-from-customer": "取消分配客户", @@ -1133,6 +1166,133 @@ "column": "列", "row": "排" }, + "edge": { + "add": "增加边缘", + "add-edge-text": "增加新的边缘", + "any-edge": "任何边缘", + "assets": "边缘资产", + "assign-edge-to-customer": "分配边缘给客户", + "assign-edge-to-customer-text": "请选择需要分配给边缘的客户", + "assign-new-edge": "分配新边缘", + "assign-to-customer": "分配给客户", + "assign-to-customer-text": "请选择需要分配给边缘的客户", + "assigned-to-customer": "分配给: {{customerTitle}}", + "assignedToCustomer": "分配给客户", + "copy-edge-key": "复制边缘键", + "copy-edge-secret": "复制边缘密钥", + "copy-id": "复制边缘编号", + "dashboard": "边缘仪表板", + "dashboards": "边缘仪表板", + "delete": "删除边缘", + "delete-edge-text": "当心, 确认后，边缘以及所有关联数据将不可恢复。", + "delete-edge-title": "您确定删除边缘 '{{edgeName}}'?", + "delete-edges-text": "当心, 确认后，选定的边缘以及所有关联数据将不可恢复。", + "delete-edges-title": "您确认删除边缘 { count, plural, 1 {1 个边缘} other {# 个边缘} }?", + "deployed": "已部署", + "description": "描述", + "details": "详情", + "devices": "边缘设备", + "downlinks": "下行", + "edge": "边缘", + "edge-assets": "边缘资产", + "edge-dashboards": "边缘仪表板", + "edge-details": "边缘详情", + "edge-devices": "边缘设备", + "edge-entity-views": "边缘实体视图", + "edge-file": "边缘文件", + "edge-instances": "边缘实例", + "edge-key": "边缘键", + "edge-key-copied-message": "边缘键已经被复制到剪切板", + "edge-public": "边缘公开", + "edge-required": "边缘必填。", + "edge-rulechains": "边缘规则链", + "edge-secret": "边缘密钥", + "edge-secret-copied-message": "边缘密钥已经被复制到剪切板", + "edge-type": "边缘类型", + "edge-type-list-empty": "没有选择边缘类型。", + "edge-type-required": "边缘类型必填。", + "edge-types": "边缘类型", + "enter-edge-type": "输入边缘类型", + "entity-id": "实体编号", + "entity-views": "边缘实体视图", + "event-action": "事件行动", + "events": "事件", + "id-copied-message": "边缘编号已经复制到剪切板", + "import": "导入边缘", + "label": "标签", + "label-max-length": "标签长度必须少于256个字符", + "load-entity-error": "加载数据失败，实体已经被删除。", + "make-private": "私有", + "make-private-edge-text": "确认后，边缘及其所有数据将被设为私有，不被其他人访问。", + "make-private-edge-title": "您确定要将边缘 '{{edgeName}}' 设为私有？", + "make-public": "公开", + "make-public-edge-text": "确认后，边缘及其所有数据将被设为公开并可被其他人访问。", + "make-public-edge-title": "您确定要将边缘 '{{edgeName}}' 设为公开吗？", + "management": "边缘管理", + "missing-related-rule-chains-text": "分配给边缘的规则链使用规则节点将消息转发给未分配给当前边缘的规则链。

缺少的规则链列表:
{{missingRuleChains}}", + "missing-related-rule-chains-title": "边缘缺少关联规则链", + "name": "名称", + "name-max-length": "名称长度必须少于256个字符", + "name-required": "名称必填。", + "name-starts-with": "边缘名称前缀", + "no-downlinks-prompt": "没有找到下行", + "no-edge-types-matching": "没有找到匹配的边缘类型 '{{entitySubtype}}'。", + "no-edges-matching": "没有找到匹配的边缘 '{{entity}}'。", + "no-edges-text": "没有找到边缘", + "pending": "待定", + "public": "公开", + "rulechain-templates": "规则链模版", + "rulechains": "规则链", + "search": "搜索边缘", + "select-edge-type": "选择边缘类型", + "selected-edges": "{ count, plural, 1 {1 个边缘} other {# 个边缘} } 被选中", + "sync": "同步边缘", + "sync-process-started-successfully": "同步处理开始成功！", + "type-max-length": "类型长度必须少于256个字符", + "unassign-edge-text": "确定后，边缘将被取消分配，并且客户将无法访问。", + "unassign-edge-title": "您确定取消分配边缘 '{{edgeName}}'?", + "unassign-edges-text": "确定后，所有选定的边缘将被取消分配，并且客户将无法访问。", + "unassign-edges-title": "您确定要取消分配 {count,plural,1 {1 个边缘} other {# 个边缘} } 吗？", + "unassign-from-customer": "取消分配客户", + "unassign-from-edge": "取消分配边缘", + "widget-datasource-error": "组件只支持边缘实体数据源" + }, + "edge-event": { + "action-type-added": "增加", + "action-type-alarm-ack": "警告确认", + "action-type-alarm-clear": "警告清除", + "action-type-assigned-to-customer": "分配给客户", + "action-type-assigned-to-edge": "分配给边缘", + "action-type-attributes-deleted": "属性删除", + "action-type-attributes-updated": "属性更新", + "action-type-credentials-request": "认证请求", + "action-type-credentials-updated": "认证更新", + "action-type-deleted": "删除", + "action-type-entity-merge-request": "实体合并请求", + "action-type-post-attributes": "推送属性", + "action-type-relation-add-or-update": "关联增加或更新", + "action-type-relation-deleted": "关联删除", + "action-type-rpc-call": "RPC调用", + "action-type-timeseries-updated": "时序更新", + "action-type-unassigned-from-customer": "取消分配客户", + "action-type-unassigned-from-edge": "取消分配边缘", + "action-type-updated": "更新", + "type-admin-settings": "管理员设置", + "type-alarm": "告警", + "type-asset": "资产", + "type-customer": "客户", + "type-dashboard": "仪表板", + "type-device": "设备", + "type-device-profile": "设备概要", + "type-edge": "边缘", + "type-entity-view": "实体视图", + "type-relation": "关联", + "type-rule-chain": "规则链", + "type-rule-chain-metadata": "规则链元数据", + "type-user": "用户", + "type-widgets-bundle": "部件包", + "type-widgets-type": "部件类型" + }, "entity-field": { "address": "地址", "address2": "地址2", @@ -1160,6 +1320,9 @@ "any-entity-view": "任何实体视图", "assign-entity-view-to-customer": "将实体视图分配给客户", "assign-entity-view-to-customer-text": "请选择要分配给客户的实体视图", + "assign-entity-view-to-edge": "将实体视图分配给边缘", + "assign-entity-view-to-edge-text": "请选择要分配给边缘的实体视图", + "assign-entity-view-to-edge-title": "将实体视图分配给边缘", "assign-entity-views": "分配实体视图", "assign-entity-views-text": "分配 { count, plural, 1 {# 个实体视图} other {# 个实体视图} } 给客户", "assign-new-entity-view": "分配新实体视图", @@ -1244,8 +1407,14 @@ "unassign-entity-view": "未分配实体视图", "unassign-entity-view-text": "确认后，实体视图将未分配，客户无法访问。", "unassign-entity-view-title": "您确定要取消对 '{{entityViewName}}'实体视图的分配吗？", + "unassign-entity-view-from-edge": "未分配实体视图", + "unassign-entity-view-from-edge-text": "确认后，实体视图将未分配，边缘无法访问。", + "unassign-entity-view-from-edge-title": "您确定要取消对 '{{entityViewName}}'实体视图的分配吗？", "unassign-entity-views": "取消分配实体视图", "unassign-entity-views-action-title": "从客户处取消分配{count,plural,1 {# 实体视图} other {# 实体视图} }", + "unassign-entity-views-from-edge-action-title": "从边缘处取消分配{count,plural,1 {# 实体视图} other {# 实体视图} }", + "unassign-entity-views-from-edge-text": "确认后，所有选定的实体视图将被分配，边缘无法访问。", + "unassign-entity-views-from-edge-title": "确定要取消分配 { count, plural, 1 {# 个实体视图} other {# 个实体视图} }吗?", "unassign-entity-views-text": "确认后，所有选定的实体视图将被分配，客户无法访问。", "unassign-entity-views-title": "确定要取消分配 { count, plural, 1 {# 个实体视图} other {# 个实体视图} }吗?", "unassign-from-customer": "取消分配客户", @@ -1361,7 +1530,11 @@ "unable-delete-entity-alias-text": "实体别名 '{{entityAlias}}' 被以下部件使用不能删除：
{{widgetsList}}", "unable-delete-entity-alias-title": "无法删除实体别名", "use-entity-name-filter": "用户筛选器", - "user-name-starts-with": "以 '{{prefix}}' 开头的用户" + "user-name-starts-with": "以 '{{prefix}}' 开头的用户", + "type-edge": "边缘", + "type-edges": "边缘", + "list-of-edges": "{ count, plural, 1 {1 个边缘} other {列表 # 个边缘} }", + "edge-name-starts-with": "以 '{{prefix}}' 开头的边缘" }, "error": { "unable-to-connect": "无法连接到服务器！请检查您的互联网连接。", @@ -1817,6 +1990,8 @@ "isgateway": "Is网关", "label": "标签", "name": "名称", + "routing-key": "边缘键", + "secret": "边缘密钥", "server-attribute": "服务器属性", "shared-attribute": "共享属性", "timeseries": "Timeseries", @@ -2087,6 +2262,9 @@ "rulechain": { "add": "添加规则链", "add-rulechain-text": "添加新的规则链", + "assign-to-edge": "分配规则链给边缘", + "assign-rulechain-to-edge-title": "分配规则链给边缘", + "assign-rulechain-to-edge-text": "请选择要分配给边缘的规则链", "copyId": "复制规则链ID", "create-new-rulechain": "创建新的规则链", "debug-mode": "调试模式", @@ -2098,12 +2276,15 @@ "delete-rulechains-title": "确实要删除{count, plural, 1 { 1 个规则链} other {# 个规则链} }吗？", "description": "说明", "details": "详情", + "edge-rulechain": "边缘规则链", + "edge-template-root": "根模版", "events": "事件", "export": "导出规则链", "export-failed-error": "无法导出规则链：{{error}}", "idCopiedMessage": "规则ID已经复制到粘贴板", "import": "导入规则链", "invalid-rulechain-file-error": "不能导入规则链：无效的规则链数据格式。", + "invalid-rulechain-type-error": "不能导入规则链：无效的规则链类型。期望类型为{{expectedRuleChainType}}。", "management": "规则集管理", "name": "名称", "name-required": "名称必填。", @@ -2119,10 +2300,22 @@ "search": "查找规则链", "select-rulechain": "选择规则链", "selected-rulechains": "已选择 { count, plural, 1 {# 个规则链} other {# 个规则链} }", + "set-auto-assign-to-edge": "创建时分配规则链给边缘", + "set-auto-assign-to-edge-text": "确认后，创建时规则链将自动分配给边缘。", + "set-auto-assign-to-edge-title": "您确定创建时将规则链'{{ruleChainName}}'自动分配给边缘吗", + "set-edge-template-root-rulechain": "设置规则链为边缘模版根", + "set-edge-template-root-rulechain-text": "确认后，规则链将将会成为边缘模版根，且它会成为新创建边缘的根规则链。", + "set-edge-template-root-rulechain-title": "您确定将规则链 '{{ruleChainName}}' 设置为边缘模版根吗？", "set-root": "设置为根规则链", "set-root-rulechain-text": "确认之后，规则链将变为根规格链，并将处理所有传入的传输消息。", "set-root-rulechain-title": "您确定要生成规则链'{{ruleChainName}}'根吗？", - "system": "系统" + "system": "系统", + "unassign-rulechain-from-edge-text": "确认后，规则链将会取消分配，边缘无法访问。", + "unassign-rulechains-from-edge-text": "确认后，选定的规则链将会取消分配，边缘无法访问。", + "unassign-rulechains-from-edge-title": "您确定要取消分配规则链 { count, plural, 1 {1 个规则链} other {# 个规则链} }?", + "unset-auto-assign-to-edge": "创建时分配规则链给边缘", + "unset-auto-assign-to-edge-text": "确认后，创建时规则链将不再自动分配给边缘。", + "unset-auto-assign-to-edge-title": "您确定取消创建时将规则链'{{ruleChainName}}'自动分配给边缘吗？" }, "rulenode": { "add": "添加规则节点", From 421624f3d48f9a0793077582d68d3fdc98197657 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 12:41:14 +0300 Subject: [PATCH 102/262] Fixed sql test data --- dao/src/test/resources/sql/system-data.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 339d6e94ba..8e38b15010 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", From 4472bd7f7c3d70f7a75a3b26bd0a81769789ce6b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 23 May 2022 12:50:59 +0300 Subject: [PATCH 103/262] Initial implementation of the ClusterVersionControlService --- .../server/actors/ActorSystemContext.java | 2 +- .../DeviceProfileMsgConstructor.java | 2 +- .../queue/DefaultTbClusterService.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 13 +- .../DefaultTbRuleEngineConsumerService.java | 3 +- .../processing/AbstractConsumerService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 63 ++--- .../DefaultGitVersionControlQueueService.java | 79 ++++++- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 9 + .../service/sync/vc/PendingGitRequest.java | 1 - .../service/sync/vc/VoidGitRequest.java | 29 +++ .../transport/DefaultTransportApiService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 + common/cluster-api/src/main/proto/queue.proto | 24 +- .../vc/EntitiesVersionControlSettings.java | 7 +- common/queue/pom.xml | 4 + .../InMemoryMonolithQueueFactory.java | 13 +- .../util/DataDecodingEncodingService.java | 2 +- .../queue}/util/ProtoWithFSTService.java | 3 +- .../service/ProtoTransportEntityService.java | 5 +- common/transport/transport-api/pom.xml | 4 - .../DefaultTransportDeviceProfileCache.java | 2 +- .../DefaultTransportResourceCache.java | 2 +- .../service/DefaultTransportService.java | 2 +- .../DefaultTransportTenantProfileCache.java | 2 +- common/version-control/pom.xml | 12 +- .../sync/vc/ClusterVersionControlService.java | 22 ++ .../DefaultClusterVersionControlService.java | 219 ++++++++++++++++++ .../sync/vc/GitVersionControlService.java | 52 ----- .../sync/vc/VersionControlRequestCtx.java | 49 ++++ 31 files changed, 505 insertions(+), 134 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/DataDecodingEncodingService.java (93%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/ProtoWithFSTService.java (93%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java delete mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 500ad60524..733cf75e64 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -48,7 +48,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java index 77dc121e38..7943ec0595 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java @@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 4a52ea2287..6b475f5a5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -140,6 +140,7 @@ public class DefaultTbClusterService implements TbClusterService { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + //TODO: ashvayka toCoreMsgs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index bec0dceed3..4cf7f86097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -74,6 +74,8 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -116,6 +118,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; @@ -137,7 +140,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { - if (!request.getFuture().isDone()) { - pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); - } else { - throw new RuntimeException("Future is already done!"); - } - } - @Override public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); @@ -218,6 +214,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu // } } + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + @Override public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -233,6 +238,56 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return request.getFuture(); } + @Override + public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture clearRepository(TenantId tenantId) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public void processResponse(VersionControlResponseMsg vcResponseMsg) { + UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); + PendingGitRequest request = pendingRequestMap.get(requestId); + if (request == null) { + log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); + return; + } else { + log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + } + var future = request.getFuture(); + if (!StringUtils.isEmpty(vcResponseMsg.getError())) { + future.setException(new RuntimeException(vcResponseMsg.getError())); + } else { + if (vcResponseMsg.hasGenericResponse()) { + future.set(null); + } + } + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 02afafedad..bb42e44f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; +import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -57,8 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); - void deleteVersionControlSettings(TenantId tenantId); + void deleteVersionControlSettings(TenantId tenantId) throws Exception; - void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index e7b7171ea0..f09b383d87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -21,10 +21,12 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import java.util.List; @@ -54,4 +56,11 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture clearRepository(TenantId tenantId); + + void processResponse(VersionControlResponseMsg vcResponseMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 3650f9e113..09bc272828 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; -import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java new file mode 100644 index 0000000000..d829960ab4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; + +import java.util.List; + +public class VoidGitRequest extends PendingGitRequest { + + public VoidGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6c4adfc5ae..f8b42ecc1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1032c79054..3fb48d2789 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1024,6 +1024,8 @@ queue: vc: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index cb1a1ab4db..7a5030277e 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -745,24 +745,34 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message GenericRepositoryRequestMsg {} + +message GenericRepositoryResponseMsg {} + message ToVersionControlServiceMsg { string nodeId = 1; int64 tenantIdMSB = 2; int64 tenantIdLSB = 3; int64 requestIdMSB = 4; int64 requestIdLSB = 5; - CommitRequestMsg commitRequest = 6; - ListVersionsRequestMsg listVersionRequest = 7; - ListEntitiesRequestMsg listEntitiesRequest = 8; - ListBranchesRequestMsg listBranchesRequest = 9; - EntityContentRequestMsg entityContentRequest = 10; - EntitiesContentRequestMsg entitiesContentRequest = 11; + bytes vcSettings = 6; + GenericRepositoryRequestMsg initRepositoryRequest = 7; + GenericRepositoryRequestMsg testRepositoryRequest = 8; + GenericRepositoryRequestMsg clearRepositoryRequest = 9; + CommitRequestMsg commitRequest = 10; + ListVersionsRequestMsg listVersionRequest = 11; + ListEntitiesRequestMsg listEntitiesRequest = 12; + ListBranchesRequestMsg listBranchesRequest = 13; + EntityContentRequestMsg entityContentRequest = 14; + EntitiesContentRequestMsg entitiesContentRequest = 15; } message VersionControlResponseMsg { int64 requestIdMSB = 1; int64 requestIdLSB = 2; - CommitResponseMsg commitResponse = 3; + string error = 3; + GenericRepositoryResponseMsg genericResponse = 4; + CommitResponseMsg commitResponse = 5; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..bc098fbe1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,8 +16,13 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + +import java.io.Serializable; + @Data -public class EntitiesVersionControlSettings { +public class EntitiesVersionControlSettings implements Serializable { + private static final long serialVersionUID = -3211552851889198721L; + private String repositoryUri; private VersionControlAuthMethod authMethod; private String username; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e2197d566f..a625ec6e93 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,6 +120,10 @@ com.google.protobuf protobuf-java-util + + de.ruedigermoeller + fst + org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 98c1a8029e..b6b89fcc2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -37,28 +37,32 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, InMemoryStorage storage) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; + this.vcSettings = vcSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -91,6 +95,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, vcSettings.getTopic()); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new InMemoryTbQueueConsumer<>(storage, configuration.getTopic()); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java index ed0572e8e5..8ebcc39cab 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import java.util.Optional; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index 1c5eec383c..f1c55973db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import java.util.Optional; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index c75de25267..a10c3e4969 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Service diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8b3f9feb22..51373e2a4a 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -64,10 +64,6 @@ com.google.code.gson gson - - de.ruedigermoeller - fst - org.slf4j slf4j-api diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index b79306199f..2b0f73343d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java index 666941b143..b4187432e8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.TransportResourceCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 5b29d3c3ff..3256039e9a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -69,7 +69,7 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4cfefb7a6f..bb7f8f3f4d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 7b46f7ba72..294141db8f 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -36,6 +36,14 @@ + + org.thingsboard.common + data + + + org.thingsboard.common + queue + org.springframework spring-core @@ -105,10 +113,6 @@ awaitility test - - org.thingsboard.common - data - diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java new file mode 100644 index 0000000000..fc134e1850 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.springframework.context.ApplicationListener; + +public interface ClusterVersionControlService extends ApplicationListener { +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java new file mode 100644 index 0000000000..73089acdf5 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -0,0 +1,219 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbVersionControlComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +@TbVersionControlComponent +@Service +@RequiredArgsConstructor +public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final TbVersionControlQueueFactory queueFactory; + private final DataDecodingEncodingService encodingService; + private final GitRepositoryService vcService; + private final NotificationsTopicService notificationsTopicService; + + private volatile ExecutorService consumerExecutor; + private volatile TbQueueConsumer> consumer; + private volatile TbQueueProducer> producer; + private volatile boolean stopped = false; + + @Value("${queue.vc.poll-interval:25}") + private long pollDuration; + @Value("${queue.vc.pack-processing-timeout:60000}") + private long packProcessingTimeout; + + @PostConstruct + public void init() { + consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); + producer = queueFactory.createTbCoreNotificationsMsgProducer(); + consumer = queueFactory.createToVersionControlMsgConsumer(); + } + + @PreDestroy + public void stop() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + //TODO: cleanup repositories that we no longer manage in this node. + consumer.subscribe(event.getPartitions()); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + consumerExecutor.execute(() -> consumerLoop(consumer)); + } + + void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msgWrapper : msgs) { + ToVersionControlServiceMsg msg = msgWrapper.getValue(); + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(new VersionControlRequestCtx(msg, null)); + } else { + VersionControlRequestCtx ctx = new VersionControlRequestCtx(msg, getEntitiesVersionControlSettings(msg)); + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } + } + } +// ConcurrentMap> pendingMap = msgs.stream().collect( +// Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); +// CountDownLatch processingTimeoutLatch = new CountDownLatch(1); +// TbPackProcessingContext> ctx = new TbPackProcessingContext<>( +// processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); +// pendingMap.forEach((id, msg) -> { +// log.trace("[{}] Creating downlink callback for message: {}", id, msg.getValue()); +// TbCallback callback = new TbPackCallback<>(id, ctx); +// try { +// handleDownlink(id, msg, callback); +// } catch (Throwable e) { +// log.warn("[{}] Failed to process notification: {}", id, msg, e); +// callback.onFailure(e); +// } +// }); +// if (!processingTimeoutLatch.await(processingTimeout, TimeUnit.MILLISECONDS)) { +// ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process downlink: {}", id, msg.getValue())); +// ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process downlink: {}", id, msg.getValue())); +// } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain version control requests from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new version control messages", e2); + } + } + } + } + log.info("TB Version Control request consumer stopped."); + } + + private void handleClearRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.clearRepository(ctx.getTenantId()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void handleInitRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + + private void handleTestRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.testRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void reply(VersionControlRequestCtx ctx, Optional e) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); + TransportProtos.VersionControlResponseMsg.Builder builder = TransportProtos.VersionControlResponseMsg.newBuilder() + .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) + .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); + if (e.isPresent()) { + builder.setError(e.get().getMessage()); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); + log.trace("PUSHING msg: {} to: {}", msg, tpi); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + if (settingsOpt.isPresent()) { + return settingsOpt.get(); + } else { + log.warn("Failed to parse VC settings: {}", msg.getVcSettings()); + throw new RuntimeException("Failed to parse vc settings!"); + } + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java deleted file mode 100644 index 56349cf8a5..0000000000 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; - -import java.util.List; - -public interface GitVersionControlService { - - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void clearRepository(TenantId tenantId); - - List listVersions(TenantId tenantId, String branch); - - List listVersions(TenantId tenantId, String branch, EntityType entityType); - - List listVersions(TenantId tenantId, String branch, EntityId entityId); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); - - List listBranches(TenantId tenantId); - - EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); - - List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); - -} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java new file mode 100644 index 0000000000..6541a2978a --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; + +import java.util.UUID; + +@RequiredArgsConstructor +@Data +public class VersionControlRequestCtx { + private final String nodeId; + private final UUID requestId; + private final TenantId tenantId; + private final EntitiesVersionControlSettings settings; + + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + this.nodeId = msg.getNodeId(); + this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); + this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + this.settings = settings; + } + + @Override + public String toString() { + return "VersionControlRequestCtx{" + + "nodeId='" + nodeId + '\'' + + ", requestId=" + requestId + + ", tenantId=" + tenantId + + '}'; + } +} From 2a89fc67aea28844c65444e3da579963c4a56254 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 13:20:00 +0300 Subject: [PATCH 104/262] Git api usage improvements --- .../DefaultEntitiesVersionControlService.java | 29 ++++++----- .../vc/LocalGitVersionControlService.java | 20 +++++++- .../src/main/resources/thingsboard.yml | 1 + .../vc/EntitiesVersionControlSettings.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 51 ++++++++++--------- .../server/service/sync/vc/GitRepository.java | 45 ++++++++-------- .../service/sync/vc/GitRepositoryService.java | 2 + .../server/service/sync/vc/PendingCommit.java | 2 + 8 files changed, 93 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d46884ca1e..4aedfb9699 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.*; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; - -import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; -import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import java.util.ArrayList; import java.util.HashMap; @@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..fa767bd1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService { if (old != null) { gitRepositoryService.abort(old); } - gitRepositoryService.prepareCommit(pendingCommit); + try { + gitRepositoryService.prepareCommit(pendingCommit); + } catch (Exception e) { + pendingCommitMap.remove(tenantId); + gitRepositoryService.cleanUp(pendingCommit); + throw e; + } return pendingCommit; } finally { lock.unlock(); @@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { @Override public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); + VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push); + pendingCommitMap.remove(commit.getTenantId()); + return result; } @Override @@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); r.accept(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } @@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); return c.apply(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 654929f500..115b4d2ebd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1119,6 +1119,7 @@ vc: git: service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: endpoints: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..308f412228 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} \ No newline at end of file +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index aac40e0dcb..e85dee8e2b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; @Service public class DefaultGitRepositoryService implements GitRepositoryService { - @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + @Value("${vc.git.repositories-folder}") private String repositoriesFolder; @Value("${vc.git.repos-poll-interval:60}") @@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService { String branch = commit.getRequest().getBranch(); try { repository.fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout("origin/" + branch, false); - try { - repository.checkout(branch, true); - } catch (RefAlreadyExistsException e) { - repository.checkout(branch, false); - } + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { repository.merge(branch); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } } } catch (IOException | GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; @@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setRemoved(status.getRemoved().size()); GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); - repository.push(); + repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch()); result.setVersion(toVersion(gitCommit)); return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); + } + } + + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); } @Override public void abort(PendingCommit commit) { - //TODO: implement; + cleanUp(commit); } @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { - GitRepository repository = checkRepository(tenantId); - return repository.getFileContentAtCommit(relativePath, versionId); + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); } @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { - return repository.listBranches(); + return repository.listRemoteBranches(); } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index fa4891d35a..5f1a13ec53 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.sshd.JGitKeyCache; @@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -116,15 +124,18 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch, boolean createBranch) throws GitAPIException { - execute(git.checkout() - .setCreateBranch(createBranch) - .setName(branch)); + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); } - public void reset() throws GitAPIException { + public void resetAndClean() throws GitAPIException { execute(git.reset() .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); } public void merge(String branch) throws IOException, GitAPIException { @@ -137,9 +148,9 @@ public class GitRepository { } - public List listBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(name -> StringUtils.removeStart(name, "origin/")) @@ -209,16 +220,9 @@ public class GitRepository { .setOrphan(true) .setForced(true) .setName(name)); -// Set uncommittedChanges = git.status().call().getUncommittedChanges(); -// if (!uncommittedChanges.isEmpty()) { -// RmCommand rm = git.rm(); -// uncommittedChanges.forEach(rm::addFilepattern); -// execute(rm); -// } -// execute(git.clean()); } - public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + public void add(String filesPattern) throws GitAPIException { execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); } @@ -230,13 +234,14 @@ public class GitRepository { public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() - .setMessage(message)); // TODO [viacheslav]: set configurable author for commit + .setMessage(message)); return toCommit(revCommit); } - public void push() throws GitAPIException { - execute(git.push()); + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 19754750c2..bbdf976804 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -44,6 +44,8 @@ public interface GitRepositoryService { VersionCreationResult push(PendingCommit commit); + void cleanUp(PendingCommit commit); + void abort(PendingCommit commit); List listBranches(TenantId tenantId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index a30f7514a2..16f3ce5e40 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -27,10 +27,12 @@ public class PendingCommit { private final UUID txId; private final TenantId tenantId; private final VersionCreateRequest request; + private final String workingBranch; public PendingCommit(TenantId tenantId, VersionCreateRequest request) { this.txId = UUID.randomUUID(); this.tenantId = tenantId; this.request = request; + this.workingBranch = txId.toString(); } } From 55966e033e75e7e7d5d13f4d55452f4698f76189 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 14:04:12 +0300 Subject: [PATCH 105/262] Fixes for vc api --- .../EntitiesVersionControlController.java | 2 +- .../sync/vc/LocalGitVersionControlService.java | 14 ++++++-------- .../common/data/sync/ie/EntityExportData.java | 3 ++- .../common/data/sync/ie/EntityImportResult.java | 6 ------ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..39a8dc464f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -223,7 +223,7 @@ public class EntitiesVersionControlController extends BaseController { if (versionId == null) { List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); if (versions.size() > 0) { - versionId = versions.get(0).getId(); + request.setVersionId(versions.get(0).getId()); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..8aa4b793df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -16,21 +16,17 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +34,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -62,13 +57,16 @@ import java.util.stream.Collectors; @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) public class LocalGitVersionControlService implements GitVersionControlService { - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); + private final ObjectMapper jsonMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { @@ -165,7 +163,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { doInsideLock(commit, c -> { String entityDataJson; try { - entityDataJson = jsonWriter.writeValueAsString(entityData); + entityDataJson = jsonMapper.writeValueAsString(entityData); gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), entityData.getEntity().getId().toString()), entityDataJson); } catch (IOException e) { @@ -230,7 +228,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + return jsonMapper.readValue(entityDataJson, EntityExportData.class); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index b361a302fc..35fceafe43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index ea69425a2c..8d6d9ff7b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -15,26 +15,20 @@ */ package org.thingsboard.server.common.data.sync.ie; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.JsonTbEntity; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { - @JsonTbEntity private E savedEntity; - @JsonTbEntity private E oldEntity; private EntityType entityType; - @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @JsonIgnore private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { From 2dcdb4af8e561fedd600b8af5cd86544251e91c1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 15:36:57 +0300 Subject: [PATCH 106/262] Fix Entities Version Control controller - add TbCoreComponent annotation --- .../server/controller/EntitiesVersionControlController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..a2b0bf3859 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -41,6 +42,7 @@ import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @RestController +@TbCoreComponent @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor From 0d57f2f6c2baf8200066a6c34b67d2bdb87a97ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 18:19:07 +0300 Subject: [PATCH 107/262] Use page link to get versions list --- .../server/controller/AdminController.java | 14 ++++++ .../EntitiesVersionControlController.java | 50 +++++++++++++------ .../DefaultEntitiesVersionControlService.java | 26 +++++----- .../vc/EntitiesVersionControlService.java | 8 +-- .../vc/LocalGitVersionControlService.java | 18 ++++--- common/version-control/pom.xml | 1 - .../sync/vc/DefaultGitRepositoryService.java | 10 ++-- .../server/service/sync/vc/GitRepository.java | 44 ++++++++++------ .../service/sync/vc/GitRepositoryService.java | 4 +- .../sync/vc/GitVersionControlService.java | 8 +-- 10 files changed, 118 insertions(+), 65 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 45b8aa5f95..ae9bf18bcf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -203,6 +203,20 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings/exists") + @ResponseBody + public Boolean versionControlSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index a2b0bf3859..4086788fea 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -25,6 +26,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @@ -39,7 +42,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -109,13 +112,18 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public PageData listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink); } catch (Exception e) { throw handleException(e); } @@ -128,11 +136,16 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public PageData listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink); } catch (Exception e) { throw handleException(e); } @@ -153,10 +166,15 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public PageData listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listVersions(getTenantId(), branch, pageLink); } catch (Exception e) { throw handleException(e); } @@ -223,9 +241,9 @@ public class EntitiesVersionControlController extends BaseController { try { String versionId = request.getVersionId(); if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); + PageData versions = versionControlService.listVersions(user.getTenantId(), request.getBranch(), new PageLink(1)); + if (versions.getData().size() > 0) { + versionId = versions.getData().get(0).getId(); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 4aedfb9699..8f0d11068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -134,18 +136,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, externalId, pageLink); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, entityType, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, pageLink); } @Override @@ -307,6 +309,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont adminSettings.setKey(SETTINGS_KEY); adminSettings.setTenantId(tenantId); } + try { + gitService.clearRepository(tenantId); + gitService.initRepository(tenantId, versionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -315,12 +323,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } return savedVersionControlSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..a70b78aaf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -37,11 +39,11 @@ public interface EntitiesVersionControlService { VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..42a1308901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -31,6 +31,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -183,18 +185,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) { + return listVersions(tenantId, branch, (String) null, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + public PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityType, null), pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + public PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), pageLink); } @Override @@ -249,9 +251,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { return null; } - private List listVersions(TenantId tenantId, String branch, String path) { + private PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) { try { - return gitRepositoryService.listVersions(tenantId, branch, path); + return gitRepositoryService.listVersions(tenantId, branch, path, pageLink); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index b01b466cb5..89c36b1c96 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -60,7 +60,6 @@ com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index e85dee8e2b..b822ff3b5b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -178,7 +180,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() + return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() .filter(version -> version.getId().equals(versionId)) .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); } @@ -189,11 +191,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); } @Override diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5f1a13ec53..22be22d991 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.LsRemoteCommand; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -45,6 +39,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -59,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class GitRepository { @@ -157,27 +154,25 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { - return listCommits(branch, null, limit); + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); } - public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } LogCommand command = git.log() - .add(branchId).setMaxCount(limit) + .add(branchId) .setRevFilter(RevFilter.NO_MERGES); if (StringUtils.isNotEmpty(path)) { command.addPath(path); } - return Streams.stream(execute(command)) - .map(this::toCommit) - .collect(Collectors.toList()); + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink); } - public List listFilesAtCommit(String commitId) throws IOException { return listFilesAtCommit(commitId, null); } @@ -295,6 +290,23 @@ public class GitRepository { return command.call(); } + private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { if (credentialsProvider != null) { transportCommand.setCredentialsProvider(credentialsProvider); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index bbdf976804..8fb0e0f84d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -28,7 +30,7 @@ public interface GitRepositoryService { void prepareCommit(PendingCommit pendingCommit); - List listVersions(TenantId tenantId, String branch, String path) throws Exception; + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..c21efbb774 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -44,11 +46,11 @@ public interface GitVersionControlService { VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityType entityType); + PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityId entityId); + PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); From baa681634b7e284090d24181ab556b39dc540ba4 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Tue, 24 May 2022 10:13:11 +0300 Subject: [PATCH 108/262] UI: Phone input with country flag select --- ui-ngx/package.json | 1 + .../components/phone-input.component.html | 51 + .../components/phone-input.component.scss | 60 + .../components/phone-input.component.ts | 215 +++ .../src/app/shared/models/country.models.ts | 1587 +++++++++++++++++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 6 + ui-ngx/yarn.lock | 5 + 8 files changed, 1930 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/shared/components/phone-input.component.html create mode 100644 ui-ngx/src/app/shared/components/phone-input.component.scss create mode 100644 ui-ngx/src/app/shared/components/phone-input.component.ts create mode 100644 ui-ngx/src/app/shared/models/country.models.ts diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 4f4316fced..feef69a2c9 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -62,6 +62,7 @@ "leaflet-providers": "^1.13.0", "leaflet.gridlayer.googlemutant": "^0.13.4", "leaflet.markercluster": "^1.5.3", + "libphonenumber-js": "^1.10.4", "messageformat": "^2.3.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", diff --git a/ui-ngx/src/app/shared/components/phone-input.component.html b/ui-ngx/src/app/shared/components/phone-input.component.html new file mode 100644 index 0000000000..9001e04ce8 --- /dev/null +++ b/ui-ngx/src/app/shared/components/phone-input.component.html @@ -0,0 +1,51 @@ + + +
+
+
+ {{ flagIcon }} + + + {{country.flag}} + {{country.name + ' +' + country.dialCode }} + + +
+ + phone-input.phone-input-label + + + {{ 'phone-input.phone-input-required' | translate }} + + + {{ 'phone-input.phone-input-validation' | translate }} + + + {{ 'phone-input.phone-input-pattern' | translate }} + + +
+
diff --git a/ui-ngx/src/app/shared/components/phone-input.component.scss b/ui-ngx/src/app/shared/components/phone-input.component.scss new file mode 100644 index 0000000000..0145a13d62 --- /dev/null +++ b/ui-ngx/src/app/shared/components/phone-input.component.scss @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + + .phone-input-container { + display: flex; + align-items: center; + + .phone-input { + width: 100%; + } + } + + .flags-select-container { + display: inline-block; + position: relative; + width: 50px; + height: 100%; + margin-right: 5px; + } + + .flag-container { + position: absolute; + font-size: 20px; + top: 50%; + left: 0; + transform: translate(0, -50%); + } + .country-select { + width: 45px; + height: 30px; + + .country-flag { + font-size: 22px; + padding-right: 10px; + } + + .mat-select-trigger { + height: 100%; + width: 100%; + } + + .mat-select-value { + visibility: hidden; + } + } +} diff --git a/ui-ngx/src/app/shared/components/phone-input.component.ts b/ui-ngx/src/app/shared/components/phone-input.component.ts new file mode 100644 index 0000000000..6618811e19 --- /dev/null +++ b/ui-ngx/src/app/shared/components/phone-input.component.ts @@ -0,0 +1,215 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + ValidatorFn, + Validators +} from '@angular/forms'; +import { TranslateService } from '@ngx-translate/core'; +import { Country, CountryData } from '@shared/models/country.models'; +import examples from 'libphonenumber-js/examples.mobile.json'; +import { CountryCode, getExampleNumber, parsePhoneNumberFromString } from 'libphonenumber-js'; +import { phoneNumberPattern } from '@shared/models/settings.models'; +import { Subscription } from 'rxjs'; + +@Component({ + selector: 'tb-phone-input', + templateUrl: './phone-input.component.html', + styleUrls: ['./phone-input.component.scss'], + providers: [ + CountryData, + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => PhoneInputComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => PhoneInputComponent), + multi: true + } + ] +}) +export class PhoneInputComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() disabled: boolean; + @Input() defaultCountry: CountryCode = 'US'; + @Input() enableFlagsSelect = true; + @Input() required = true; + + allCountries: Array = []; + phonePlaceholder: string; + countryCallingCode: string; + flagIcon: string; + + phoneNumberPattern = phoneNumberPattern; + + private modelValue: string; + public phoneFormGroup: FormGroup; + private valueChange$: Subscription = null; + private propagateChange = (v: any) => { }; + + constructor(private translate: TranslateService, + private fb: FormBuilder, + private countryCodeData: CountryData) { + } + + ngOnInit(): void { + if (this.enableFlagsSelect) { + this.fetchCountryData(); + } + this.phoneFormGroup = this.fb.group({ + country: [this.defaultCountry, []], + phoneNumber: [ + this.countryCallingCode, + this.required ? [Validators.required, Validators.pattern(phoneNumberPattern), this.validatePhoneNumber()] : [] + ] + }); + + this.valueChange$ = this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(() => { + this.updateModel(); + }); + + this.phoneFormGroup.get('country').valueChanges.subscribe(value => { + if (value) { + const code = this.countryCallingCode; + this.getFlagAndPhoneNumberData(value); + let phoneNumber = this.phoneFormGroup.get('phoneNumber').value; + if (code !== this.countryCallingCode && phoneNumber) { + phoneNumber = phoneNumber.replace(code, this.countryCallingCode); + } else { + phoneNumber = this.countryCallingCode; + } + this.phoneFormGroup.get('phoneNumber').patchValue(phoneNumber); + } + }); + + this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(value => { + if (value) { + const parsedPhoneNumber = parsePhoneNumberFromString(value); + const country = this.phoneFormGroup.get('country').value; + if (parsedPhoneNumber?.country && parsedPhoneNumber?.country !== country) { + this.phoneFormGroup.get('country').patchValue(parsedPhoneNumber.country, {emitEvent: true}); + } + } + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + } + + getFlagAndPhoneNumberData(country) { + if (this.enableFlagsSelect) { + this.flagIcon = this.getFlagIcon(country); + } + this.getPhoneNumberData(country); + } + + getPhoneNumberData(country): void { + const phoneData = getExampleNumber(country, examples); + this.phonePlaceholder = phoneData.number; + this.countryCallingCode = '+' + phoneData.countryCallingCode; + } + + getFlagIcon(countryCode) { + const base = 127462 - 65; + return String.fromCodePoint(...countryCode.split('').map(country => base + country.charCodeAt(0))); + } + + validatePhoneNumber(): ValidatorFn { + return (c: FormControl) => { + const phoneNumber = c.value; + if (phoneNumber) { + const parsedPhoneNumber = parsePhoneNumberFromString(phoneNumber); + if (!parsedPhoneNumber?.isValid() || !parsedPhoneNumber?.isPossible()) { + return { + invalidPhoneNumber: { + valid: false + } + }; + } + } + return null; + }; + } + + validate(): ValidationErrors | null { + return this.phoneFormGroup.get('phoneNumber').valid ? null : { + phoneFormGroup: false + }; + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.phoneFormGroup.disable({emitEvent: false}); + } else { + this.phoneFormGroup.enable({emitEvent: false}); + } + } + + protected fetchCountryData(): void { + this.allCountries = []; + this.countryCodeData.allCountries.forEach((c) => { + const cc = c[1]; + const country: Country = { + name: c[0].toString(), + iso2: c[1].toString(), + dialCode: c[2].toString(), + flag: this.getFlagIcon(cc), + }; + + this.allCountries.push(country); + }); + } + + writeValue(phoneNumber): void { + this.modelValue = phoneNumber; + const country = phoneNumber ? parsePhoneNumberFromString(phoneNumber)?.country : this.defaultCountry; + this.getFlagAndPhoneNumberData(country); + this.phoneFormGroup.patchValue({phoneNumber, country}, {emitEvent: !phoneNumber}); + } + + private updateModel() { + const phoneNumber = this.phoneFormGroup.get('phoneNumber'); + if (phoneNumber.valid && this.modelValue !== phoneNumber.value) { + this.modelValue = phoneNumber.value; + this.propagateChange(this.modelValue); + } else { + this.propagateChange(null); + } + } + +} diff --git a/ui-ngx/src/app/shared/models/country.models.ts b/ui-ngx/src/app/shared/models/country.models.ts new file mode 100644 index 0000000000..a4ed4d38d9 --- /dev/null +++ b/ui-ngx/src/app/shared/models/country.models.ts @@ -0,0 +1,1587 @@ +import { Injectable } from '@angular/core'; + +export interface Country { + name: string; + iso2: string; + dialCode: string; + areaCodes?: string[]; + flag: string; +} + +export enum SearchCountryField { + DialCode = 'dialCode', + Iso2 = 'iso2', + Name = 'name', + All = 'all' +} + + +export enum CountryISO{ + Afghanistan = 'AF', + Albania = 'AL', + Algeria = 'DZ', + AmericanSamoa = 'AS', + Andorra = 'AD', + Angola = 'AO', + Anguilla = 'AI', + AntiguaAndBarbuda = 'AG', + Argentina = 'AR', + Armenia = 'AM', + Aruba = 'AW', + Australia = 'AU', + Austria = 'AT', + Azerbaijan = 'AZ', + Bahamas = 'BS', + Bahrain = 'BH', + Bangladesh = 'BD', + Barbados = 'BB', + Belarus = 'BY', + Belgium = 'BE', + Belize = 'BZ', + Benin = 'BJ', + Bermuda = 'BM', + Bhutan = 'BT', + Bolivia = 'BO', + BosniaAndHerzegovina = 'BA', + Botswana = 'BW', + Brazil = 'BR', + BritishIndianOceanTerritory = 'IO', + BritishVirginIslands = 'VG', + Brunei = 'BN', + Bulgaria = 'BG', + BurkinaFaso = 'BF', + Burundi = 'BI', + Cambodia = 'KH', + Cameroon = 'CM', + Canada = 'CA', + CapeVerde = 'CV', + CaribbeanNetherlands = 'BQ', + CaymanIslands = 'KY', + CentralAfricanRepublic = 'CF', + Chad = 'TD', + Chile = 'CL', + China = 'CN', + ChristmasIsland = 'CX', + Cocos = 'CC', + Colombia = 'CC', + Comoros = 'KM', + CongoDRCJamhuriYaKidemokrasiaYaKongo = 'CD', + CongoRepublicCongoBrazzaville = 'CG', + CookIslands = 'CK', + CostaRica = 'CR', + CôteDIvoire = 'CI', + Croatia = 'HR', + Cuba = 'CU', + Curaçao = 'CW', + Cyprus = 'CY', + CzechRepublic = 'CZ', + Denmark = 'DK', + Djibouti = 'DJ', + Dominica = 'DM', + DominicanRepublic = 'DO', + Ecuador = 'EC', + Egypt = 'EG', + ElSalvador = 'SV', + EquatorialGuinea = 'GQ', + Eritrea = 'ER', + Estonia = 'EE', + Ethiopia = 'ET', + FalklandIslands = 'FK', + FaroeIslands = 'FO', + Fiji = 'FJ', + Finland = 'FI', + France = 'FR', + FrenchGuiana = 'GF', + FrenchPolynesia = 'PF', + Gabon = 'GA', + Gambia = 'GM', + Georgia = 'GE', + Germany = 'DE', + Ghana = 'GH', + Gibraltar = 'GI', + Greece = 'GR', + Greenland = 'GL', + Grenada = 'GD', + Guadeloupe = 'GP', + Guam = 'GU', + Guatemala = 'GT', + Guernsey = 'GG', + Guinea = 'GN', + GuineaBissau = 'GW', + Guyana = 'GY', + Haiti = 'HT', + Honduras = 'HN', + HongKong = 'HK', + Hungary = 'HU', + Iceland = 'IS', + India = 'IN', + Indonesia = 'ID', + Iran = 'IR', + Iraq = 'IQ', + Ireland = 'IE', + IsleOfMan = 'IM', + Israel = 'IL', + Italy = 'IT', + Jamaica = 'JM', + Japan = 'JP', + Jersey = 'JE', + Jordan = 'JO', + Kazakhstan = 'KZ', + Kenya = 'KE', + Kiribati = 'KI', + Kosovo = 'XK', + Kuwait = 'KW', + Kyrgyzstan = 'KG', + Laos = 'LA', + Latvia = 'LV', + Lebanon = 'LB', + Lesotho = 'LS', + Liberia = 'LR', + Libya = 'LY', + Liechtenstein = 'LI', + Lithuania = 'LT', + Luxembourg = 'LU', + Macau = 'MO', + Macedonia = 'MK', + Madagascar = 'MG', + Malawi = 'MW', + Malaysia = 'MY', + Maldives = 'MV', + Mali = 'ML', + Malta = 'MT', + MarshallIslands = 'MH', + Martinique = 'MQ', + Mauritania = 'MR', + Mauritius = 'MU', + Mayotte = 'YT', + Mexico = 'MX', + Micronesia = 'FM', + Moldova = 'MD', + Monaco = 'MC', + Mongolia = 'MN', + Montenegro = 'ME', + Montserrat = 'MS', + Morocco = 'MA', + Mozambique = 'MZ', + Myanmar = 'MM', + Namibia = 'NA', + Nauru = 'NR', + Nepal = 'NP', + Netherlands = 'NL', + NewCaledonia = 'NC', + NewZealand = 'NZ', + Nicaragua = 'NI', + Niger = 'NE', + Nigeria = 'NG', + Niue = 'NU', + NorfolkIsland = 'NF', + NorthKorea = 'KP', + NorthernMarianaIslands = 'MP', + Norway = 'NO', + Oman = 'OM', + Pakistan = 'PK', + Palau = 'PW', + Palestine = 'PS', + Panama = 'PA', + PapuaNewGuinea = 'PG', + Paraguay = 'PY', + Peru = 'PE', + Philippines = 'PH', + Poland = 'PL', + Portugal = 'PT', + PuertoRico = 'PR', + Qatar = 'QA', + Réunion = 'RE', + Romania = 'RO', + Russia = 'RU', + Rwanda = 'RW', + SaintBarthélemy = 'BL', + SaintHelena = 'SH', + SaintKittsAndNevis = 'KN', + SaintLucia = 'LC', + SaintMartin = 'MF', + SaintPierreAndMiquelon = 'PM', + SaintVincentAndTheGrenadines = 'VC', + Samoa = 'WS', + SanMarino = 'SM', + SãoToméAndPríncipe = 'ST', + SaudiArabia = 'SA', + Senegal = 'SN', + Serbia = 'RS', + Seychelles = 'SC', + SierraLeone = 'SL', + Singapore = 'SG', + SintMaarten = 'SX', + Slovakia = 'SK', + Slovenia = 'SI', + SolomonIslands = 'SB', + Somalia = 'SO', + SouthAfrica = 'ZA', + SouthKorea = 'KR', + SouthSudan = 'SS', + Spain = 'ES', + SriLanka = 'LK', + Sudan = 'SD', + Suriname = 'SR', + SvalbardAndJanMayen = 'SJ', + Swaziland = 'SZ', + Sweden = 'SE', + Switzerland = 'CH', + Syria = 'SY', + Taiwan = 'TW', + Tajikistan = 'TJ', + Tanzania = 'TZ', + Thailand = 'TH', + TimorLeste = 'TL', + Togo = 'TG', + Tokelau = 'TK', + Tonga = 'TO', + TrinidadAndTobago = 'TT', + Tunisia = 'TN', + Turkey = 'TR', + Turkmenistan = 'TM', + TurksAndCaicosIslands = 'TC', + Tuvalu = 'TV', + USVirginIslands = 'VI', + Uganda = 'UG', + Ukraine = 'UA', + UnitedArabEmirates = 'AE', + UnitedKingdom = 'GB', + UnitedStates = 'US', + Uruguay = 'UY', + Uzbekistan = 'UZ', + Vanuatu = 'VU', + VaticanCity = 'VA', + Venezuela = 'VE', + Vietnam = 'VN', + WallisAndFutuna = 'WF', + WesternSahara = 'EH', + Yemen = 'YE', + Zambia = 'ZM', + Zimbabwe = 'ZW', + ÅlandIslands = 'AX', +} + +@Injectable() +export class CountryData { + public allCountries = [ + [ + 'Afghanistan', + CountryISO.Afghanistan, + '93' + ], + [ + 'Albania', + CountryISO.Albania, + '355' + ], + [ + 'Algeria', + CountryISO.Algeria, + '213' + ], + [ + 'American Samoa', + CountryISO.AmericanSamoa, + '1', + 1, + [ + '684', + ] + ], + [ + 'Andorra', + CountryISO.Andorra, + '376' + ], + [ + 'Angola', + CountryISO.Angola, + '244' + ], + [ + 'Anguilla', + CountryISO.Anguilla, + '1', + 1, + [ + '264', + ] + ], + [ + 'Antigua and Barbuda', + CountryISO.AntiguaAndBarbuda, + '1', + 1, + [ + '268', + ] + ], + [ + 'Argentina', + CountryISO.Argentina, + '54' + ], + [ + 'Armenia', + CountryISO.Armenia, + '374' + ], + [ + 'Aruba', + CountryISO.Aruba, + '297' + ], + [ + 'Australia', + CountryISO.Australia, + '61', + 0 + ], + [ + 'Austria', + CountryISO.Austria, + '43' + ], + [ + 'Azerbaijan', + CountryISO.Azerbaijan, + '994' + ], + [ + 'Bahamas', + CountryISO.Bahamas, + '1', + 1, + [ + '242', + ] + ], + [ + 'Bahrain', + CountryISO.Bahrain, + '973' + ], + [ + 'Bangladesh', + CountryISO.Bangladesh, + '880' + ], + [ + 'Barbados', + CountryISO.Barbados, + '1', + 1, + [ + '246', + ] + ], + [ + 'Belarus', + CountryISO.Belarus, + '375' + ], + [ + 'Belgium', + CountryISO.Belgium, + '32' + ], + [ + 'Belize', + CountryISO.Belize, + '501' + ], + [ + 'Benin', + CountryISO.Benin, + '229' + ], + [ + 'Bermuda', + CountryISO.Bermuda, + '1', + 1, + [ + '441', + ] + ], + [ + 'Bhutan', + CountryISO.Bhutan, + '975' + ], + [ + 'Bolivia', + CountryISO.Bolivia, + '591' + ], + [ + 'Bosnia and Herzegovina', + CountryISO.BosniaAndHerzegovina, + '387' + ], + [ + 'Botswana', + CountryISO.Botswana, + '267' + ], + [ + 'Brazil', + CountryISO.Brazil, + '55' + ], + [ + 'British Indian Ocean Territory', + CountryISO.BritishIndianOceanTerritory, + '246' + ], + [ + 'British Virgin Islands', + CountryISO.BritishVirginIslands, + '1', + 1, + [ + '284', + ] + ], + [ + 'Brunei', + CountryISO.Brunei, + '673' + ], + [ + 'Bulgaria', + CountryISO.Bulgaria, + '359' + ], + [ + 'Burkina Faso', + CountryISO.BurkinaFaso, + '226' + ], + [ + 'Burundi', + CountryISO.Burundi, + '257' + ], + [ + 'Cambodia', + CountryISO.Cambodia, + '855' + ], + [ + 'Cameroon', + CountryISO.Cameroon, + '237' + ], + [ + 'Canada', + CountryISO.Canada, + '1', + 1, + [ + '204', '226', '236', '249', '250', '289', '306', '343', '365', '387', '403', '416', + '418', '431', '437', '438', '450', '506', '514', '519', '548', '579', '581', '587', + '604', '613', '639', '647', '672', '705', '709', '742', '778', '780', '782', '807', + '819', '825', '867', '873', '902', '905' + ] + ], + [ + 'Cape Verde', + CountryISO.CapeVerde, + '238' + ], + [ + 'Caribbean Netherlands', + CountryISO.CaribbeanNetherlands, + '599', + 1 + ], + [ + 'Cayman Islands', + CountryISO.CaymanIslands, + '1', + 1, + [ + '345', + ] + ], + [ + 'Central African Republic', + CountryISO.CentralAfricanRepublic, + '236' + ], + [ + 'Chad', + CountryISO.Chad, + '235' + ], + [ + 'Chile', + CountryISO.Chile, + '56' + ], + [ + 'China', + CountryISO.China, + '86' + ], + [ + 'Christmas Island', + CountryISO.ChristmasIsland, + '61', + 2 + ], + [ + 'Cocos Islands', + CountryISO.Cocos, + '61', + 1 + ], + [ + 'Colombia', + CountryISO.Colombia, + '57' + ], + [ + 'Comoros', + CountryISO.Comoros, + '269' + ], + [ + 'Congo-Kinshasa', + CountryISO.CongoDRCJamhuriYaKidemokrasiaYaKongo, + '243' + ], + [ + 'Congo-Brazzaville', + CountryISO.CongoRepublicCongoBrazzaville, + '242' + ], + [ + 'Cook Islands', + CountryISO.CookIslands, + '682' + ], + [ + 'Costa Rica', + CountryISO.CostaRica, + '506' + ], + [ + 'Côte d’Ivoire', + CountryISO.CôteDIvoire, + '225' + ], + [ + 'Croatia', + CountryISO.Croatia, + '385' + ], + [ + 'Cuba', + CountryISO.Cuba, + '53' + ], + [ + 'Curaçao', + CountryISO.Curaçao, + '599', + 0 + ], + [ + 'Cyprus', + CountryISO.Cyprus, + '357' + ], + [ + 'Czech Republic', + CountryISO.CzechRepublic, + '420' + ], + [ + 'Denmark', + CountryISO.Denmark, + '45' + ], + [ + 'Djibouti', + CountryISO.Djibouti, + '253' + ], + [ + 'Dominica', + CountryISO.Dominica, + '1767' + ], + [ + 'Dominican Republic', + CountryISO.DominicanRepublic, + '1', + 2, + ['809', '829', '849'] + ], + [ + 'Ecuador', + CountryISO.Ecuador, + '593' + ], + [ + 'Egypt', + CountryISO.Egypt, + '20' + ], + [ + 'El Salvador', + CountryISO.ElSalvador, + '503' + ], + [ + 'Equatorial Guinea', + CountryISO.EquatorialGuinea, + '240' + ], + [ + 'Eritrea', + CountryISO.Eritrea, + '291' + ], + [ + 'Estonia', + CountryISO.Estonia, + '372' + ], + [ + 'Ethiopia', + CountryISO.Ethiopia, + '251' + ], + [ + 'Falkland Islands', + CountryISO.FalklandIslands, + '500' + ], + [ + 'Faroe Islands', + CountryISO.FaroeIslands, + '298' + ], + [ + 'Fiji', + CountryISO.Fiji, + '679' + ], + [ + 'Finland', + CountryISO.Finland, + '358', + 0 + ], + [ + 'France', + CountryISO.France, + '33' + ], + [ + 'French Guiana', + CountryISO.FrenchGuiana, + '594' + ], + [ + 'French Polynesia', + CountryISO.FrenchPolynesia, + '689' + ], + [ + 'Gabon', + CountryISO.Gabon, + '241' + ], + [ + 'Gambia', + CountryISO.Gambia, + '220' + ], + [ + 'Georgia', + CountryISO.Georgia, + '995' + ], + [ + 'Germany', + CountryISO.Germany, + '49' + ], + [ + 'Ghana', + CountryISO.Ghana, + '233' + ], + [ + 'Gibraltar', + CountryISO.Gibraltar, + '350' + ], + [ + 'Greece', + CountryISO.Greece, + '30' + ], + [ + 'Greenland', + CountryISO.Greenland, + '299' + ], + [ + 'Grenada', + CountryISO.Grenada, + '1473' + ], + [ + 'Guadeloupe', + CountryISO.Guadeloupe, + '590', + 0 + ], + [ + 'Guam', + CountryISO.Guam, + '1', + 1, + [ + '671', + ] + ], + [ + 'Guatemala', + CountryISO.Guatemala, + '502' + ], + [ + 'Guernsey', + CountryISO.Guernsey, + '44', + 1, + [1481] + ], + [ + 'Guinea', + CountryISO.Guinea, + '224' + ], + [ + 'Guinea-Bissau', + CountryISO.GuineaBissau, + '245' + ], + [ + 'Guyana', + CountryISO.Guyana, + '592' + ], + [ + 'Haiti', + CountryISO.Haiti, + '509' + ], + [ + 'Honduras', + CountryISO.Honduras, + '504' + ], + [ + 'Hong Kong', + CountryISO.HongKong, + '852' + ], + [ + 'Hungary', + CountryISO.Hungary, + '36' + ], + [ + 'Iceland', + CountryISO.Iceland, + '354' + ], + [ + 'India', + CountryISO.India, + '91' + ], + [ + 'Indonesia', + CountryISO.Indonesia, + '62' + ], + [ + 'Iran', + CountryISO.Iran, + '98' + ], + [ + 'Iraq', + CountryISO.Iraq, + '964' + ], + [ + 'Ireland', + CountryISO.Ireland, + '353' + ], + [ + 'Isle of Man', + CountryISO.IsleOfMan, + '44', + 2, + [1624] + ], + [ + 'Israel', + CountryISO.Israel, + '972' + ], + [ + 'Italy', + CountryISO.Italy, + '39', + 0 + ], + [ + 'Jamaica', + CountryISO.Jamaica, + '1', + 1, + [ + '876', + ] + ], + [ + 'Japan', + CountryISO.Japan, + '81' + ], + [ + 'Jersey', + CountryISO.Jersey, + '44', + 3, + [1534] + ], + [ + 'Jordan', + CountryISO.Jordan, + '962' + ], + [ + 'Kazakhstan', + CountryISO.Kazakhstan, + '7', + 1 + ], + [ + 'Kenya', + CountryISO.Kenya, + '254' + ], + [ + 'Kiribati', + CountryISO.Kiribati, + '686' + ], + [ + 'Kosovo', + CountryISO.Kosovo, + '383' + ], + [ + 'Kuwait', + CountryISO.Kuwait, + '965' + ], + [ + 'Kyrgyzstan', + CountryISO.Kyrgyzstan, + '996' + ], + [ + 'Laos', + CountryISO.Laos, + '856' + ], + [ + 'Latvia', + CountryISO.Latvia, + '371' + ], + [ + 'Lebanon', + CountryISO.Lebanon, + '961' + ], + [ + 'Lesotho', + CountryISO.Lesotho, + '266' + ], + [ + 'Liberia', + CountryISO.Liberia, + '231' + ], + [ + 'Libya', + CountryISO.Libya, + '218' + ], + [ + 'Liechtenstein', + CountryISO.Liechtenstein, + '423' + ], + [ + 'Lithuania', + CountryISO.Lithuania, + '370' + ], + [ + 'Luxembourg', + CountryISO.Luxembourg, + '352' + ], + [ + 'Macau', + CountryISO.Macau, + '853' + ], + [ + 'Macedonia', + CountryISO.Macedonia, + '389' + ], + [ + 'Madagascar', + CountryISO.Madagascar, + '261' + ], + [ + 'Malawi', + CountryISO.Malawi, + '265' + ], + [ + 'Malaysia', + CountryISO.Malaysia, + '60' + ], + [ + 'Maldives', + CountryISO.Maldives, + '960' + ], + [ + 'Mali', + CountryISO.Mali, + '223' + ], + [ + 'Malta', + CountryISO.Malta, + '356' + ], + [ + 'Marshall Islands', + CountryISO.MarshallIslands, + '692' + ], + [ + 'Martinique', + CountryISO.Martinique, + '596' + ], + [ + 'Mauritania', + CountryISO.Mauritania, + '222' + ], + [ + 'Mauritius', + CountryISO.Mauritius, + '230' + ], + [ + 'Mayotte', + CountryISO.Mayotte, + '262', + 1 + ], + [ + 'Mexico', + CountryISO.Mexico, + '52' + ], + [ + 'Micronesia', + CountryISO.Micronesia, + '691' + ], + [ + 'Moldova', + CountryISO.Moldova, + '373' + ], + [ + 'Monaco', + CountryISO.Monaco, + '377' + ], + [ + 'Mongolia', + CountryISO.Mongolia, + '976' + ], + [ + 'Montenegro', + CountryISO.Montenegro, + '382' + ], + [ + 'Montserrat', + CountryISO.Montserrat, + '1', + 1, + [ + '664', + ] + ], + [ + 'Morocco', + CountryISO.Morocco, + '212', + 0 + ], + [ + 'Mozambique', + CountryISO.Mozambique, + '258' + ], + [ + 'Myanmar', + CountryISO.Myanmar, + '95' + ], + [ + 'Namibia', + CountryISO.Namibia, + '264' + ], + [ + 'Nauru', + CountryISO.Nauru, + '674' + ], + [ + 'Nepal', + CountryISO.Nepal, + '977' + ], + [ + 'Netherlands', + CountryISO.Netherlands, + '31' + ], + [ + 'New Caledonia', + CountryISO.NewCaledonia, + '687' + ], + [ + 'New Zealand', + CountryISO.NewZealand, + '64' + ], + [ + 'Nicaragua', + CountryISO.Nicaragua, + '505' + ], + [ + 'Niger', + CountryISO.Niger, + '227' + ], + [ + 'Nigeria', + CountryISO.Nigeria, + '234' + ], + [ + 'Niue', + CountryISO.Niue, + '683' + ], + [ + 'Norfolk Island', + CountryISO.NorfolkIsland, + '672' + ], + [ + 'North Korea', + CountryISO.NorthKorea, + '850' + ], + [ + 'Northern Mariana Islands', + CountryISO.NorthernMarianaIslands, + '1670' + ], + [ + 'Norway', + CountryISO.Norway, + '47', + 0 + ], + [ + 'Oman', + CountryISO.Oman, + '968' + ], + [ + 'Pakistan', + CountryISO.Pakistan, + '92' + ], + [ + 'Palau', + CountryISO.Palau, + '680' + ], + [ + 'Palestine', + CountryISO.Palestine, + '970' + ], + [ + 'Panama', + CountryISO.Panama, + '507' + ], + [ + 'Papua New Guinea', + CountryISO.PapuaNewGuinea, + '675' + ], + [ + 'Paraguay', + CountryISO.Paraguay, + '595' + ], + [ + 'Peru', + CountryISO.Peru, + '51' + ], + [ + 'Philippines', + CountryISO.Philippines, + '63' + ], + [ + 'Poland', + CountryISO.Poland, + '48' + ], + [ + 'Portugal', + CountryISO.Portugal, + '351' + ], + [ + 'Puerto Rico', + CountryISO.PuertoRico, + '1', + 3, + ['787', '939'] + ], + [ + 'Qatar', + CountryISO.Qatar, + '974' + ], + [ + 'Réunion', + CountryISO.Réunion, + '262', + 0 + ], + [ + 'Romania', + CountryISO.Romania, + '40' + ], + [ + 'Russia', + CountryISO.Russia, + '7', + 0 + ], + [ + 'Rwanda', + CountryISO.Rwanda, + '250' + ], + [ + 'Saint Barthélemy', + CountryISO.SaintBarthélemy, + '590', + 1 + ], + [ + 'Saint Helena', + CountryISO.SaintHelena, + '290' + ], + [ + 'Saint Kitts and Nevis', + CountryISO.SaintKittsAndNevis, + '1869' + ], + [ + 'Saint Lucia', + CountryISO.SaintLucia, + '1', + 1, + [ + '758', + ] + ], + [ + 'Saint Martin', + CountryISO.SaintMartin, + '590', + 2 + ], + [ + 'Saint Pierre and Miquelon', + CountryISO.SaintPierreAndMiquelon, + '508' + ], + [ + 'Saint Vincent and the Grenadines', + CountryISO.SaintVincentAndTheGrenadines, + '1', + 1, + [ + '784', + ] + ], + [ + 'Samoa', + CountryISO.Samoa, + '685' + ], + [ + 'San Marino', + CountryISO.SanMarino, + '378' + ], + [ + 'São Tomé and Príncipe', + CountryISO.SãoToméAndPríncipe, + '239' + ], + [ + 'Saudi Arabia', + CountryISO.SaudiArabia, + '966' + ], + [ + 'Senegal', + CountryISO.Senegal, + '221' + ], + [ + 'Serbia', + CountryISO.Serbia, + '381' + ], + [ + 'Seychelles', + CountryISO.Seychelles, + '248' + ], + [ + 'Sierra Leone', + CountryISO.SierraLeone, + '232' + ], + [ + 'Singapore', + CountryISO.Singapore, + '65' + ], + [ + 'Sint Maarten', + CountryISO.SintMaarten, + '1', + 1, + [ + '721', + ] + ], + [ + 'Slovakia', + CountryISO.Slovakia, + '421' + ], + [ + 'Slovenia', + CountryISO.Slovenia, + '386' + ], + [ + 'Solomon Islands', + CountryISO.SolomonIslands, + '677' + ], + [ + 'Somalia', + CountryISO.Somalia, + '252' + ], + [ + 'South Africa', + CountryISO.SouthAfrica, + '27' + ], + [ + 'South Korea', + CountryISO.SouthKorea, + '82' + ], + [ + 'South Sudan', + CountryISO.SouthSudan, + '211' + ], + [ + 'Spain', + CountryISO.Spain, + '34' + ], + [ + 'Sri Lanka', + CountryISO.SriLanka, + '94' + ], + [ + 'Sudan', + CountryISO.Sudan, + '249' + ], + [ + 'Suriname', + CountryISO.Suriname, + '597' + ], + [ + 'Svalbard and Jan Mayen', + CountryISO.SvalbardAndJanMayen, + '47', + 1 + ], + [ + 'Swaziland', + CountryISO.Swaziland, + '268' + ], + [ + 'Sweden', + CountryISO.Sweden, + '46' + ], + [ + 'Switzerland', + CountryISO.Switzerland, + '41' + ], + [ + 'Syria', + CountryISO.Syria, + '963' + ], + [ + 'Taiwan', + CountryISO.Taiwan, + '886' + ], + [ + 'Tajikistan', + CountryISO.Tajikistan, + '992' + ], + [ + 'Tanzania', + CountryISO.Tanzania, + '255' + ], + [ + 'Thailand', + CountryISO.Thailand, + '66' + ], + [ + 'Timor-Leste', + CountryISO.TimorLeste, + '670' + ], + [ + 'Togo', + CountryISO.Togo, + '228' + ], + [ + 'Tokelau', + CountryISO.Tokelau, + '690' + ], + [ + 'Tonga', + CountryISO.Tonga, + '676' + ], + [ + 'Trinidad and Tobago', + CountryISO.TrinidadAndTobago, + '1', + 1, + [ + '868', + ] + ], + [ + 'Tunisia', + CountryISO.Tunisia, + '216' + ], + [ + 'Turkey', + CountryISO.Turkey, + '90' + ], + [ + 'Turkmenistan', + CountryISO.Turkmenistan, + '993' + ], + [ + 'Turks and Caicos Islands', + CountryISO.TurksAndCaicosIslands, + '1649' + ], + [ + 'Tuvalu', + CountryISO.Tuvalu, + '688' + ], + [ + 'U.S. Virgin Islands', + CountryISO.USVirginIslands, + '1', + 1, + [ + '340', + ] + ], + [ + 'Uganda', + CountryISO.Uganda, + '256' + ], + [ + 'Ukraine', + CountryISO.Ukraine, + '380' + ], + [ + 'United Arab Emirates', + CountryISO.UnitedArabEmirates, + '971' + ], + [ + 'United Kingdom', + CountryISO.UnitedKingdom, + '44', + 0 + ], + [ + 'United States', + CountryISO.UnitedStates, + '1', + 0 + ], + [ + 'Uruguay', + CountryISO.Uruguay, + '598' + ], + [ + 'Uzbekistan', + CountryISO.Uzbekistan, + '998' + ], + [ + 'Vanuatu', + CountryISO.Vanuatu, + '678' + ], + [ + 'Vatican City', + CountryISO.VaticanCity, + '39', + 1 + ], + [ + 'Venezuela', + CountryISO.Venezuela, + '58' + ], + [ + 'Vietnam', + CountryISO.Vietnam, + '84' + ], + [ + 'Wallis and Futuna', + CountryISO.WallisAndFutuna, + '681' + ], + [ + 'Western Sahara', + CountryISO.WesternSahara, + '212', + 1 + ], + [ + 'Yemen', + CountryISO.Yemen, + '967' + ], + [ + 'Zambia', + CountryISO.Zambia, + '260' + ], + [ + 'Zimbabwe', + CountryISO.Zimbabwe, + '263' + ], + [ + 'Åland Islands', + CountryISO.ÅlandIslands, + '358', + 1 + ] + ]; +} diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index a356fa5921..706ffb16a0 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -159,6 +159,7 @@ import { TbMarkdownComponent } from '@shared/components/markdown.component'; import { ProtobufContentComponent } from '@shared/components/protobuf-content.component'; import { CssComponent } from '@shared/components/css.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; +import { PhoneInputComponent } from '@shared/components/phone-input.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -277,7 +278,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + PhoneInputComponent ], imports: [ CommonModule, @@ -472,7 +474,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + PhoneInputComponent ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index aab433c992..050080d5ba 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3351,6 +3351,12 @@ "material-icons": "Material icons", "show-all": "Show all icons" }, + "phone-input": { + "phone-input-label": "Phone number", + "phone-input-required": "Phone number is required", + "phone-input-validation": "Phone number is invalid or not possible", + "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. +19995550123." + }, "custom": { "widget-action": { "action-cell-button": "Action cell button", diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index 0aac92b131..45fe658406 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -6027,6 +6027,11 @@ less@4.1.1: needle "^2.5.2" source-map "~0.6.0" +libphonenumber-js@^1.10.4: + version "1.10.4" + resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.10.4.tgz#90397f0ed620262570a32244c9fbc389cc417ce4" + integrity sha512-9QWxEk4GW5RDnFzt8UtyRENfFpAN8u7Sbf9wf32tcXY9tdtnz1dKHIBwW2Wnfx8ypXJb9zUnTpK9aQJ/B8AlnA== + license-webpack-plugin@2.3.20: version "2.3.20" resolved "https://registry.yarnpkg.com/license-webpack-plugin/-/license-webpack-plugin-2.3.20.tgz#f51fb674ca31519dbedbe1c7aabc036e5a7f2858" From 1065fd9fbebec782ed7fddc268f26d712cd10917 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 11:38:03 +0300 Subject: [PATCH 109/262] Implementation of the API --- .../DefaultGitVersionControlQueueService.java | 49 ++- .../vc/LocalGitVersionControlService.java | 290 ------------------ .../service/sync/vc/PendingGitRequest.java | 4 +- common/cluster-api/src/main/proto/queue.proto | 37 +++ .../data/sync/vc/VersionedEntityInfo.java | 6 + .../DefaultClusterVersionControlService.java | 109 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../service/sync/vc/GitRepositoryService.java | 4 +- 8 files changed, 188 insertions(+), 319 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7ffcbb9c14..0d3f2a4431 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -21,17 +21,16 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -61,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.function.Function; +import java.util.stream.Collectors; @TbCoreComponent @Service @@ -209,6 +209,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override + @SuppressWarnings("rawtypes") public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() @@ -218,14 +219,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() , wrap(request.getFuture())); return request.getFuture(); -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } } private void registerAndSend(PendingGitRequest request, @@ -237,13 +230,16 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request, settings)), callback); + var requestBody = enrichFunction.apply(newRequestProto(request, settings)); + log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); + clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); } else { throw new RuntimeException("Future is already done!"); } } @Override + @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -313,10 +309,41 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); ((CommitGitRequest) request).getFuture().set(commitResult); + } else if (vcResponseMsg.hasListBranchesResponse()) { + var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + } else if (vcResponseMsg.hasListEntitiesResponse()) { + var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); + ((ListEntitiesGitRequest) request).getFuture().set( + listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListVersionsResponse()) { + var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); + ((ListVersionsGitRequest) request).getFuture().set( + listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList())); + } else if (vcResponseMsg.hasEntityContentResponse()) { + var data = vcResponseMsg.getEntityContentResponse().getData(); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else if (vcResponseMsg.hasEntitiesContentResponse()) { + var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); + ((EntitiesContentGitRequest) request).getFuture() + .set(dataList.stream().map(this::toData).collect(Collectors.toList())); } } } + private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { + return new EntityVersion(proto.getId(), proto.getName()); + } + + private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { + return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); + } + + @SuppressWarnings("rawtypes") + private EntityExportData toData(String data) { + return JacksonUtil.fromString(data, EntityExportData.class); + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java deleted file mode 100644 index d995472531..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; - -import java.io.IOException; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -@RequiredArgsConstructor -@Service -@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService { - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private final GitRepositoryService gitRepositoryService; - private final TenantDao tenantDao; - private final AdminSettingsService adminSettingsService; - private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); -// -// @AfterStartUp -// public void init() { -// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { -// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); -// if (settings != null) { -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// log.warn("Failed to init repository for tenant {}", tenantId, e); -// } -// } -// }); -// } -// -// @Override -// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.testRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void clearRepository(TenantId tenantId) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.clearRepository(tenantId); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// var pendingCommit = new PendingCommit(tenantId, request); -// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); -// if (old != null) { -// gitRepositoryService.abort(old); -// } -// gitRepositoryService.prepareCommit(pendingCommit); -// return pendingCommit; -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void deleteAll(PendingCommit commit, EntityType entityType) { -// doInsideLock(commit, c -> { -// try { -// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { -// doInsideLock(commit, c -> { -// String entityDataJson; -// try { -// entityDataJson = jsonWriter.writeValueAsString(entityData); -// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), -// entityData.getEntity().getId().toString()), entityDataJson); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public VersionCreationResult push(PendingCommit commit) { -// return executeInsideLock(commit, gitRepositoryService::push); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch) { -// return listVersions(tenantId, branch, (String) null); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { -// return listVersions(tenantId, branch, getRelativePath(entityType, null)); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { -// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { -// try { -// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { -// return listEntitiesAtVersion(tenantId, branch, versionId, null); -// } -// -// @Override -// public List listBranches(TenantId tenantId) { -// return gitRepositoryService.listBranches(tenantId); -// } -// -// @Override -// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { -// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() -// .skip(offset).limit(limit) -// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) -// .collect(Collectors.toList()); -// } -// -// @Override -// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { -// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); -// if (adminSettings != null) { -// try { -// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); -// } catch (Exception e) { -// throw new RuntimeException("Failed to load version control settings!", e); -// } -// } -// return null; -// } -// -// private List listVersions(TenantId tenantId, String branch, String path) { -// try { -// return gitRepositoryService.listVersions(tenantId, branch, path); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private void doInsideLock(PendingCommit commit, Consumer r) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// r.accept(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private T executeInsideLock(PendingCommit commit, Function c) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// return c.apply(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private void checkCommit(PendingCommit commit) { -// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); -// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { -// throw new ConcurrentModificationException(); -// } -// } -// -// private String getRelativePath(EntityType entityType, String entityId) { -// String path = entityType.name().toLowerCase(); -// if (entityId != null) { -// path += "/" + entityId + ".json"; -// } -// return path; -// } -// -// private Lock getRepoLock(TenantId tenantId) { -// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); -// } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 9b612cee5b..e63ba04f45 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -24,17 +24,19 @@ import java.util.UUID; @Getter public class PendingGitRequest { + private final long createdTime; private final UUID requestId; private final TenantId tenantId; private final SettableFuture future; public PendingGitRequest(TenantId tenantId) { + this.createdTime = System.currentTimeMillis(); this.requestId = UUID.randomUUID(); this.tenantId = tenantId; this.future = SettableFuture.create(); } - public boolean requiresSettings(){ + public boolean requiresSettings() { return true; } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 6d45698451..54e46eebcc 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -723,15 +723,39 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; } +message EntityVersionProto { + int64 ts = 1; + string id = 2; + string name = 3; +} + +message ListVersionsResponseMsg { + repeated EntityVersionProto versions = 1; +} + message ListEntitiesRequestMsg { string branchName = 1; string versionId = 2; string entityType = 3; } +message VersionedEntityInfoProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; +} + +message ListEntitiesResponseMsg { + repeated VersionedEntityInfoProto entities = 1; +} + message ListBranchesRequestMsg { } +message ListBranchesResponseMsg { + repeated string branches = 1; +} + message EntityContentRequestMsg { string versionId = 1; string entityType = 2; @@ -739,6 +763,10 @@ message EntityContentRequestMsg { int64 entityIdLSB = 4; } +message EntityContentResponseMsg { + string data = 1; +} + message EntitiesContentRequestMsg { string versionId = 1; string entityType = 2; @@ -746,6 +774,10 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message EntitiesContentResponseMsg { + repeated string data = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -774,6 +806,11 @@ message VersionControlResponseMsg { string error = 3; GenericRepositoryResponseMsg genericResponse = 4; CommitResponseMsg commitResponse = 5; + ListBranchesResponseMsg listBranchesResponse = 6; + ListEntitiesResponseMsg listEntitiesResponse = 7; + ListVersionsResponseMsg listVersionsResponse = 8; + EntityContentResponseMsg entityContentResponse = 9; + EntitiesContentResponseMsg entitiesContentResponse = 10; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 163fe4c6d2..fd278cde1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -16,10 +16,16 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; @Data +@NoArgsConstructor public class VersionedEntityInfo { private EntityId externalId; // etc.. + + public VersionedEntityInfo(EntityId externalId) { + this.externalId = externalId; + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b9c595beac..b282799e24 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -23,9 +23,12 @@ import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -58,6 +61,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @TbVersionControlComponent @@ -144,9 +148,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var newSettings = ctx.getSettings(); if (!newSettings.equals(currentSettings)) { vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } else { + vcService.fetch(ctx.getTenantId()); } if (msg.hasCommitRequest()) { handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); } } } @@ -172,12 +188,71 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } - private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg commitRequest) throws Exception { + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { + var entityType = EntityType.valueOf(request.getEntityType()); + String path = getRelativePath(entityType, null); + var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) + .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); + var response = EntitiesContentResponseMsg.newBuilder(); + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + response.addData(data); + } + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + } + + private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { + String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + } + + private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { + String path; + if (StringUtils.isNotEmpty(request.getEntityType())) { + var entityType = EntityType.valueOf(request.getEntityType()); + if (request.getEntityIdLSB() != 0 || request.getEntityIdMSB() != 0) { + path = getRelativePath(entityType, new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + } else { + path = getRelativePath(entityType, null); + } + } else { + path = null; + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() + .addAllVersions(data.stream().map( + v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + ).collect(Collectors.toList())))); + } + + private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { + EntityType entityType = StringUtils.isNotEmpty(request.getEntityType()) ? EntityType.valueOf(request.getEntityType()) : null; + var path = entityType != null ? getRelativePath(entityType, null) : null; + var data = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListEntitiesResponse(ListEntitiesResponseMsg.newBuilder() + .addAllEntities(data.stream().map(VersionedEntityInfo::getExternalId).map( + id -> VersionedEntityInfoProto.newBuilder() + .setEntityType(id.getEntityType().name()) + .setEntityIdMSB(id.getId().getMostSignificantBits()) + .setEntityIdLSB(id.getId().getLeastSignificantBits()).build() + ).collect(Collectors.toList())))); + } + + private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { + var branches = vcService.listBranches(ctx.getTenantId()); + reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); + } + + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); - UUID txId = UUID.fromString(commitRequest.getTxId()); - if (commitRequest.hasPrepareMsg()) { - prepareCommit(ctx, txId, commitRequest.getPrepareMsg()); - } else if (commitRequest.hasAbortMsg()) { + UUID txId = UUID.fromString(request.getTxId()); + if (request.hasPrepareMsg()) { + prepareCommit(ctx, txId, request.getPrepareMsg()); + } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { doAbortCurrentCommit(tenantId, current); @@ -186,11 +261,11 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { try { - if (commitRequest.hasAddMsg()) { - addToCommit(ctx, current, commitRequest.getAddMsg()); - } else if (commitRequest.hasDeleteMsg()) { - deleteFromCommit(ctx, current, commitRequest.getDeleteMsg()); - } else if (commitRequest.hasPushMsg()) { + if (request.hasAddMsg()) { + addToCommit(ctx, current, request.getAddMsg()); + } else if (request.hasDeleteMsg()) { + deleteFromCommit(ctx, current, request.getDeleteMsg()); + } else if (request.hasPushMsg()) { reply(ctx, vcService.push(current)); } } catch (Exception e) { @@ -198,7 +273,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe throw e; } } else { - log.debug("[{}] Ignore request due to stale commit: {}", txId, commitRequest); + log.debug("[{}] Ignore request due to stale commit: {}", txId, request); } } } @@ -290,7 +365,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); } ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); - log.trace("PUSHING msg: {} to: {}", msg, tpi); + log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @@ -304,9 +379,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } - - } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 0d55783a31..758641f6eb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -160,6 +160,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { //TODO: implement; } + @Override + public void fetch(TenantId tenantId) { + //Fetch latest changes on demand. + } + @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { GitRepository repository = checkRepository(tenantId); @@ -197,9 +202,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + public List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index c748147856..54ab91582e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -30,7 +30,7 @@ public interface GitRepositoryService { List listVersions(TenantId tenantId, String branch, String path) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; @@ -51,4 +51,6 @@ public interface GitRepositoryService { List listBranches(TenantId tenantId); String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + + void fetch(TenantId tenantId); } From a5ff23a0a479275f35f4e443497f78daf0eec4d0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 24 May 2022 14:03:53 +0300 Subject: [PATCH 110/262] Add commit timestamp and timestamp sorting --- .../EntitiesVersionControlController.java | 29 +++++++++++----- .../DefaultGitVersionControlQueueService.java | 4 +-- common/cluster-api/src/main/proto/queue.proto | 11 ++++--- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 33 +++++++++++++++---- 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index ef1d402c42..0efa52dee2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -53,9 +53,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; @RestController @TbCoreComponent @@ -131,10 +130,14 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); } catch (Exception e) { throw handleException(e); @@ -154,9 +157,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); } catch (Exception e) { throw handleException(e); @@ -183,9 +190,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink)); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fb6ab3e0d6..fbee0502e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -312,7 +312,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -344,7 +344,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index bd8e220a18..a1cbddf6e8 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -689,11 +689,12 @@ message CommitRequestMsg { } message CommitResponseMsg { - string commitId = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; + int64 ts = 1; + string commitId = 2; + string name = 3; + int32 added = 4; + int32 modified = 5; + int32 removed = 6; } message PrepareMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 8779d64ed3..50d3fef8a8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -23,6 +23,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor public class EntityVersion { + private long timestamp; private String id; private String name; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..47d0aca292 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -245,7 +245,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() ).collect(Collectors.toList()))) ); } @@ -364,6 +364,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() + .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) .setAdded(result.getAdded()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..8d711c3e69 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -259,7 +259,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } private EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dcefe7bed7..6208b913f4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; @@ -48,6 +49,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -58,10 +60,7 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PublicKey; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -180,7 +179,7 @@ public class GitRepository { command.addPath(path); } Iterable commits = execute(command); - return iterableToPageData(commits, this::toCommit, pageLink); + return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); } public List listFilesAtCommit(String commitId) throws IOException { @@ -282,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -300,11 +299,30 @@ public class GitRepository { return command.call(); } - private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + private static Function> revCommitComparatorFunction = pageLink -> { + SortOrder sortOrder = pageLink.getSortOrder(); + if (sortOrder != null + && sortOrder.getProperty().equals("timestamp") + && SortOrder.Direction.ASC.equals(sortOrder.getDirection())) { + return Comparator.comparingInt(RevCommit::getCommitTime); + } + return null; + }; + + private static PageData iterableToPageData (Iterable iterable, + Function mapper, + PageLink pageLink, + Function> comparatorFunction) { int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); int limit = startIndex + pageLink.getPageSize(); + if (comparatorFunction != null) { + Comparator comparator = comparatorFunction.apply(pageLink); + if (comparator != null) { + iterable = Ordering.from(comparator).immutableSortedCopy(iterable); + } + } iterable = Iterables.limit(iterable, limit); if (startIndex < totalElements) { iterable = Iterables.skip(iterable, startIndex); @@ -374,6 +392,7 @@ public class GitRepository { @Data public static class Commit { + private final long timestamp; private final String id; private final String message; private final String authorName; From 9deb67afe7ecddfc7ad72c89430c1a6cb959cde1 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 17:32:29 +0300 Subject: [PATCH 111/262] Version Control microservice --- .../src/main/resources/thingsboard.yml | 8 +- .../queue/discovery/ZkDiscoveryService.java | 2 - .../queue/settings/TbQueueCoreSettings.java | 2 + .../TbQueueRemoteJsInvokeSettings.java | 2 + .../settings/TbQueueRuleEngineSettings.java | 2 + .../settings/TbQueueTransportApiSettings.java | 2 + .../TbQueueTransportNotificationSettings.java | 2 + .../TbQueueVersionControlSettings.java | 2 + .../DefaultClusterVersionControlService.java | 156 ++++++++++---- .../sync/vc/DefaultGitRepositoryService.java | 37 ++-- .../service/sync/vc/GitRepositoryService.java | 3 + msa/pom.xml | 1 + msa/vc-executor-docker/docker/Dockerfile | 33 +++ .../docker/start-tb-vc-executor.sh | 33 +++ msa/vc-executor-docker/pom.xml | 190 ++++++++++++++++++ msa/vc-executor/pom.xml | 6 +- ...oardVersionControlExecutorApplication.java | 6 +- ...VersionControlQueueRoutingInfoService.java | 31 +++ ...ersionControlTenantRoutingInfoService.java | 30 +++ .../src/main/resources/logback.xml | 1 + .../src/main/resources/tb-vc-executor.yml | 160 ++++++++++++++- 21 files changed, 632 insertions(+), 77 deletions(-) create mode 100644 msa/vc-executor-docker/docker/Dockerfile create mode 100755 msa/vc-executor-docker/docker/start-tb-vc-executor.sh create mode 100644 msa/vc-executor-docker/pom.xml create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 76cc62d09a..aaec0852ca 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1025,7 +1025,7 @@ queue: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1120,9 +1120,11 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: - thread_pool_size: "${TB_VC_POOL_SIZE:4}" + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" git: - repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 8afd4dc768..16af4befd4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,14 +33,12 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 33e6552eb9..f64143b14c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueCoreSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index b930d04312..f4d62bad1d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRemoteJsInvokeSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eee70e9e22..57131f6b62 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 0ad9414bd8..a4dfe86bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportApiSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index d4c0064692..e2a10b31b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportNotificationSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java index e7f0ce2259..3a7a5602d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueVersionControlSettings { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..c4bb637eb0 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -15,8 +15,14 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; @@ -57,9 +63,10 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbVersionControlComponent; @@ -67,6 +74,7 @@ import org.thingsboard.server.queue.util.TbVersionControlComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +84,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -87,7 +97,8 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { - private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; private final TbVersionControlQueueFactory queueFactory; private final DataDecodingEncodingService encodingService; private final GitRepositoryService vcService; @@ -105,11 +116,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private long pollDuration; @Value("${queue.vc.pack-processing-timeout:60000}") private long packProcessingTimeout; + @Value("${vc.git.io_pool_size:3}") + private int ioPoolSize; + + //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. + private final List ioThreads = new ArrayList<>(); + @PostConstruct public void init() { consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); - producer = queueFactory.createTbCoreNotificationsMsgProducer(); + var threadFactory = ThingsBoardThreadFactory.forName("vc-io-thread"); + for (int i = 0; i < ioPoolSize; i++) { + ioThreads.add(MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(threadFactory))); + } + producer = producerProvider.getTbCoreNotificationsMsgProducer(); consumer = queueFactory.createToVersionControlMsgConsumer(); } @@ -122,11 +143,25 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe if (consumerExecutor != null) { consumerExecutor.shutdownNow(); } + ioThreads.forEach(ExecutorService::shutdownNow); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - //TODO: cleanup repositories that we no longer manage in this node. + for (TenantId tenantId : vcService.getActiveRepositoryTenants()) { + if (!partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId).isMyPartition()) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + pendingCommitMap.remove(tenantId); + vcService.clearRepository(tenantId); + } catch (Exception e) { + log.warn("[{}] Failed to cleanup the tenant repository", tenantId, e); + } finally { + lock.unlock(); + } + } + } consumer.subscribe(event.getPartitions()); } @@ -143,6 +178,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe void consumerLoop(TbQueueConsumer> consumer) { while (!stopped && !consumer.isStopped()) { + List> futures = new ArrayList<>(); try { List> msgs = consumer.poll(pollDuration); if (msgs.isEmpty()) { @@ -151,46 +187,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe for (TbProtoQueueMsg msgWrapper : msgs) { ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); - var lock = getRepoLock(ctx.getTenantId()); - lock.lock(); - try { - if (msg.hasClearRepositoryRequest()) { - handleClearRepositoryCommand(ctx); - } else { - if (msg.hasTestRepositoryRequest()) { - handleTestRepositoryCommand(ctx); - } else if (msg.hasInitRepositoryRequest()) { - handleInitRepositoryCommand(ctx); - } else { - var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); - var newSettings = ctx.getSettings(); - if (!newSettings.equals(currentSettings)) { - vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); - } else { - vcService.fetch(ctx.getTenantId()); - } - if (msg.hasCommitRequest()) { - handleCommitRequest(ctx, msg.getCommitRequest()); - } else if (msg.hasListBranchesRequest()) { - handleListBranches(ctx, msg.getListBranchesRequest()); - } else if (msg.hasListEntitiesRequest()) { - handleListEntities(ctx, msg.getListEntitiesRequest()); - } else if (msg.hasListVersionRequest()) { - handleListVersions(ctx, msg.getListVersionRequest()); - } else if (msg.hasEntityContentRequest()) { - handleEntityContentRequest(ctx, msg.getEntityContentRequest()); - } else if (msg.hasEntitiesContentRequest()) { - handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); - } - } - } - } catch (Exception e) { - reply(ctx, Optional.of(e)); - } finally { - lock.unlock(); - } + long startTs = System.currentTimeMillis(); + log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); + ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + logTaskExecution(ctx, future, startTs); + futures.add(future); + } + try { + Futures.allAsList(futures).get(packProcessingTimeout, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Timeout for processing the version control tasks.", e); } - //TODO: handle timeouts and async processing for multiple tenants; consumer.commit(); } catch (Exception e) { if (!stopped) { @@ -206,6 +213,51 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } + private Void processMessage(VersionControlRequestCtx ctx, ToVersionControlServiceMsg msg) { + var lock = getRepoLock(ctx.getTenantId()); + lock.lock(); + try { + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(ctx); + } else { + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } else { + var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); + var newSettings = ctx.getSettings(); + if (!newSettings.equals(currentSettings)) { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } + if (msg.hasCommitRequest()) { + handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } + } + } + } catch (Exception e) { + reply(ctx, Optional.of(e)); + } finally { + lock.unlock(); + } + return null; + } + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { var entityType = EntityType.valueOf(request.getEntityType()); String path = getRelativePath(entityType, null); @@ -273,6 +325,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); if (request.hasPrepareMsg()) { + vcService.fetch(ctx.getTenantId()); prepareCommit(ctx, txId, request.getPrepareMsg()); } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); @@ -414,4 +467,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } + + private void logTaskExecution(VersionControlRequestCtx ctx, ListenableFuture future, long startTs) { + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback() { + + @Override + public void onSuccess(@Nullable Object result) { + log.trace("[{}][{}] Task processing took: {}ms", ctx.getTenantId(), ctx.getRequestId(), (System.currentTimeMillis() - startTs)); + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}][{}] Task failed: ", ctx.getTenantId(), ctx.getRequestId(), t); + } + }, MoreExecutors.directExecutor()); + } + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..e30dd8aa20 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -35,19 +35,17 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -61,10 +59,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") private String repositoriesFolder; - @Value("${vc.git.repos-poll-interval:60}") - private long reposPollInterval; - - private ScheduledExecutorService scheduler; private final Map repositories = new ConcurrentHashMap<>(); @PostConstruct @@ -72,24 +66,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { if (StringUtils.isEmpty(repositoriesFolder)) { repositoriesFolder = defaultFolder; } - scheduler = Executors.newSingleThreadScheduledExecutor(); - scheduler.scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); } - @PreDestroy - public void stop() { - if (scheduler != null) { - scheduler.shutdownNow(); - } + @Override + public Set getActiveRepositoryTenants() { + return new HashSet<>(repositories.keySet()); } @Override @@ -151,6 +132,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @SneakyThrows @Override public void cleanUp(PendingCommit commit) { + log.debug("[{}] Cleanup tenant repository started.", commit.getTenantId()); GitRepository repository = checkRepository(commit.getTenantId()); try { repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); @@ -161,6 +143,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } repository.resetAndClean(); repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + log.debug("[{}] Cleanup tenant repository completed.", commit.getTenantId()); } @Override @@ -172,7 +155,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void fetch(TenantId tenantId) throws GitAPIException { var repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Fetching tenant repository.", tenantId); repository.fetch(); + log.debug("[{}] Fetched tenant repository.", tenantId); } } @@ -232,6 +217,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Override public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { clearRepository(tenantId); + log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; if (Files.exists(repositoryDirectory)) { @@ -241,6 +227,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings, repositoryDirectory.toFile()); repositories.put(tenantId, repository); + log.debug("[{}] Init tenant repository completed.", tenantId); } @Override @@ -253,8 +240,10 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Clear tenant repository started.", tenantId); FileUtils.deleteDirectory(new File(repository.getDirectory())); repositories.remove(tenantId); + log.debug("[{}] Clear tenant repository completed.", tenantId); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 075e08a33e..9d57eee3ec 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -26,9 +26,12 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.io.IOException; import java.util.List; +import java.util.Set; public interface GitRepositoryService { + Set getActiveRepositoryTenants(); + void prepareCommit(PendingCommit pendingCommit); PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; diff --git a/msa/pom.xml b/msa/pom.xml index 888291502b..dd40348bc8 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -41,6 +41,7 @@ tb vc-executor + vc-executor-docker js-executor web-ui tb-node diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile new file mode 100644 index 0000000000..c2f9640a6b --- /dev/null +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk11 + +COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin + +RUN yes | dpkg -i /tmp/${pkg.name}.deb +RUN rm /tmp/${pkg.name}.deb + +RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : + +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-vc-executor.sh"] diff --git a/msa/vc-executor-docker/docker/start-tb-vc-executor.sh b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh new file mode 100755 index 0000000000..4384008fd3 --- /dev/null +++ b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml new file mode 100644 index 0000000000..39fc35f32a --- /dev/null +++ b/msa/vc-executor-docker/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor-docker + pom + + ThingsBoard Version Control Executor Microservice + https://thingsboard.io + ThingsBoard Version Control Executor Microservice + + + UTF-8 + ${basedir}/../.. + tb-vc-executor + tb-vc-executor + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard.msa + vc-executor + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-vc-executor-deb + package + + copy + + + + + org.thingsboard.msa + vc-executor + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml index 7e4df36117..0125354371 100644 --- a/msa/vc-executor/pom.xml +++ b/msa/vc-executor/pom.xml @@ -38,7 +38,7 @@ false process-resources package - tb-mqtt-transport + tb-vc-executor false ${project.build.directory}/windows ThingsBoard Version Control Executor Service @@ -46,6 +46,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common version-control diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index d30818110d..6978f94b51 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -15,17 +15,17 @@ package org.thingsboard.server.vc; /** */ import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; import java.util.Arrays; -@SpringBootConfiguration +@SpringBootApplication @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) +@ComponentScan({"org.thingsboard.server", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java new file mode 100644 index 0000000000..0f4ac69948 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class VersionControlQueueRoutingInfoService implements QueueRoutingInfoService { + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java new file mode 100644 index 0000000000..ebd7ed07b3 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class VersionControlTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + //This dummy implementation is ok since Version Control service does not produce any rule engine messages. + return new TenantRoutingInfo(tenantId, false, false); + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 2834934ee0..572d093dde 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -25,6 +25,7 @@ + diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 1de47e2750..8c34a91e80 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -23,5 +23,163 @@ server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). - port: "${HTTP_BIND_PORT:8080}" + port: "${HTTP_BIND_PORT:8086}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +queue: + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + - key: max.poll.records + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + ota: + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +service: + type: "${TB_SERVICE_TYPE:tb-vc-executor}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" \ No newline at end of file From dbec05a2c173ec1446170c2300b757b1e0d05819 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 10:39:48 +0300 Subject: [PATCH 112/262] Test fix --- .../service/cluster/routing/HashPartitionServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java index 74738c361f..6aafa833b1 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java @@ -71,6 +71,8 @@ public class HashPartitionServiceTest { queueRoutingInfoService); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10); + ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); + ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() .setServiceId("tb-core-0") From ca3c95afca1e55290e82cbe4e1a213402c4c9a57 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 12:26:23 +0300 Subject: [PATCH 113/262] UI: Implement entitiy versions table --- .../EntitiesVersionControlController.java | 7 +- .../DefaultGitVersionControlQueueService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 3 +- ui-ngx/src/app/core/auth/auth.actions.ts | 11 +- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 6 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 5 + ui-ngx/src/app/core/auth/auth.service.ts | 12 +- .../http/entities-version-control.service.ts | 27 +- ui-ngx/src/app/core/services/menu.service.ts | 17 ++ .../home/components/home-components.module.ts | 13 +- .../vc/entity-versions-table.component.html | 81 ++++++ .../vc/entity-versions-table.component.scss | 100 +++++++ .../vc/entity-versions-table.component.ts | 244 ++++++++++++++++++ .../version-control-settings.component.html | 2 +- .../version-control-settings.component.scss | 3 + .../vc}/version-control-settings.component.ts | 53 ++-- .../vc/version-control.component.html | 25 ++ .../vc/version-control.component.scss | 18 ++ .../vc/version-control.component.ts | 61 +++++ .../home/pages/admin/admin-routing.module.ts | 4 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ...sion-control-admin-settings.component.html | 18 ++ ...ersion-control-admin-settings.component.ts | 44 ++++ .../pages/device/device-tabs.component.html | 5 + .../modules/home/pages/home-pages.module.ts | 4 +- .../home/pages/vc/vc-routing.module.ts | 44 ++++ .../app/modules/home/pages/vc/vc.module.ts | 31 +++ .../vc/branch-autocomplete.component.html | 5 +- .../vc/branch-autocomplete.component.ts | 147 ++++++++--- ui-ngx/src/app/shared/models/asset.models.ts | 4 +- ui-ngx/src/app/shared/models/base-data.ts | 6 + .../src/app/shared/models/customer.model.ts | 3 +- .../src/app/shared/models/dashboard.models.ts | 4 +- ui-ngx/src/app/shared/models/device.models.ts | 6 +- .../app/shared/models/rule-chain.models.ts | 4 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 10 +- 38 files changed, 948 insertions(+), 87 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.html (98%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.scss (95%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.ts (82%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc.module.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0efa52dee2..bafc93c19f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -294,11 +294,14 @@ public class EntitiesVersionControlController extends BaseController { String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + remoteBranches.forEach(branch -> { + if (!branch.equals(defaultBranch)) { + infos.add(new BranchInfo(branch, false)); + } + }); return infos; }, MoreExecutors.directExecutor())); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fbee0502e2..f738582bb3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -411,7 +411,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } if (vcSettings != null) { builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings))); - } else { + } else if (request.requiresSettings()) { throw new RuntimeException("No entity version control settings provisioned!"); } return builder; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 6208b913f4..20ed03795d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -170,7 +170,7 @@ public class GitRepository { public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); + return new PageData<>(); } LogCommand command = git.log() .add(branchId) @@ -313,6 +313,7 @@ public class GitRepository { Function mapper, PageLink pageLink, Function> comparatorFunction) { + iterable = Streams.stream(iterable).collect(Collectors.toList()); int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 6edbdcd5a2..872607b822 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -23,7 +23,8 @@ export enum AuthActionTypes { UNAUTHENTICATED = '[Auth] Unauthenticated', LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', - UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id' + UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', + UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' } export class ActionAuthAuthenticated implements Action { @@ -54,5 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } +export class ActionAuthUpdateHasVersionControl implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; + + constructor(readonly payload: { hasVersionControl: boolean }) {} +} + export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 9267d48e28..a69b783526 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,6 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; + hasVersionControl: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index be2712300d..9d9404524b 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -23,7 +23,8 @@ const emptyUserAuthState: AuthPayload = { userTokenAccessEnabled: false, forceFullscreen: false, allowedDashboardIds: [], - edgesSupportEnabled: false + edgesSupportEnabled: false, + hasVersionControl: false }; export const initialState: AuthState = { @@ -54,6 +55,9 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; + case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + return { ...state, ...action.payload}; + default: return state; } diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index aaadf00ec3..7a099d5acd 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,6 +55,11 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); +export const selectHasVersionControl = createSelector( + selectAuthState, + (state: AuthState) => state.hasVersionControl +); + export function getCurrentAuthState(store: Store): AuthState { let state: AuthState; store.pipe(select(selectAuth), take(1)).subscribe( diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6b129da6ad..6fe809955f 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,17 +437,27 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } + private loadHasVersionControl(authUser: AuthUser): Observable { + if (authUser.authority === Authority.TENANT_ADMIN) { + return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + } else { + return of(false); + } + } + private loadSystemParams(authPayload: AuthPayload): Observable { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), + this.loadHasVersionControl(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled}; + const hasVersionControl: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 231581a02f..f8668f2ecf 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,12 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { Observable } from 'rxjs'; -import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { PageData } from '@shared/models/page/page-data'; +import { DeviceInfo } from '@shared/models/device.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; @Injectable({ providedIn: 'root' @@ -37,4 +42,24 @@ export class EntitiesVersionControlService { public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); } + + public listEntityVersions(pageLink: PageLink, branch: string, + externalEntityId: EntityId, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listEntityTypeVersions(pageLink: PageLink, branch: string, + entityType: EntityType, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listVersions(pageLink: PageLink, branch: string, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 56191ec948..ee81f007ee 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -343,6 +343,13 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, + { + id: guid(), + name: 'version-control.version-control', + type: 'link', + path: '/vc', + icon: 'history' + }, { id: guid(), name: 'audit-log.audit-logs', @@ -492,6 +499,16 @@ export class MenuService { } ] }, + { + name: 'version-control.management', + places: [ + { + name: 'version-control.version-control', + icon: 'history', + path: '/vc' + } + ] + }, { name: 'audit-log.audit', places: [ diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 05067e78ee..49f1f57fd1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -154,6 +154,9 @@ import { QueueFormComponent } from '@home/components/queue/queue-form.component' import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; +import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; @NgModule({ declarations: @@ -278,7 +281,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], imports: [ CommonModule, @@ -397,7 +403,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html new file mode 100644 index 0000000000..b47c2fcc79 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -0,0 +1,81 @@ + +
+
+ +
+
+ {{(singleEntityMode ? 'version-control.entity-versions' : 'version-control.versions') | translate}} + + +
+ + +
+ +
+ + + {{ 'version-control.created-time' | translate }} + + {{ entityVersion.timestamp | date:'yyyy-MM-dd HH:mm:ss' }} + + + + {{ 'version-control.version-id' | translate }} + + {{ entityVersion.id }} + + + + {{ 'version-control.version-name' | translate }} + + {{ entityVersion.name }} + + + + +
+ {{ + singleEntityMode + ? 'version-control.no-entity-versions-text' + : 'version-control.no-versions-text' + }} +
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss new file mode 100644 index 0000000000..9017de60b8 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + width: 100%; + height: 100%; + display: block; + .tb-entity-table { + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; + + .mat-toolbar-tools{ + min-height: auto; + } + + .title-container{ + overflow: hidden; + } + + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .table-container { + overflow: auto; + } + + .tb-entity-table-info{ + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .button-widget-action{ + margin-left: auto; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + @media #{$mat-xs} { + .mat-toolbar { + height: auto; + min-height: 100px; + + .tb-entity-table-title{ + padding-bottom: 5px; + width: 100%; + } + } + } +} + +:host ::ng-deep { + .mat-sort-header-sorted .mat-sort-header-arrow { + opacity: 1 !important; + } + tb-branch-autocomplete { + mat-form-field { + font-size: 16px; + width: 200px; + + .mat-form-field-wrapper { + padding-bottom: 0; + } + + .mat-form-field-underline { + bottom: 0; + } + + @media #{$mat-xs} { + width: 100%; + + .mat-form-field-infix { + width: auto !important; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts new file mode 100644 index 0000000000..4fc33beafb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -0,0 +1,244 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + Input, + OnDestroy, + OnInit, + ViewChild +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityId } from '@shared/models/id/entity-id'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, map, tap } from 'rxjs/operators'; +import { EntityVersion } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { ResizeObserver } from '@juggle/resize-observer'; +import { hidePageSizePixelValue } from '@shared/models/constants'; +import { Direction, SortOrder } from '@shared/models/page/sort-order'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; + +@Component({ + selector: 'tb-entity-versions-table', + templateUrl: './entity-versions-table.component.html', + styleUrls: ['./entity-versions-table.component.scss'] +}) +export class EntityVersionsTableComponent extends PageComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('branchAutocompleteComponent') branchAutocompleteComponent: BranchAutocompleteComponent; + + @Input() + singleEntityMode = false; + + displayedColumns = ['timestamp', 'id', 'name']; + pageLink: PageLink; + dataSource: EntityVersionsDatasource; + hidePageSize = false; + + branch: string = null; + + activeValue = false; + dirtyValue = false; + externalEntityIdValue: EntityId; + + viewsInited = false; + + private componentResize$: ResizeObserver; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + if (this.viewsInited) { + this.initFromDefaultBranch(); + } + } + } + } + + @Input() + set externalEntityId(externalEntityId: EntityId) { + if (this.externalEntityIdValue !== externalEntityId) { + this.externalEntityIdValue = externalEntityId; + this.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.dirtyValue = true; + } + } + } + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private elementRef: ElementRef) { + super(store); + this.dirtyValue = !this.activeValue; + const sortOrder: SortOrder = { property: 'timestamp', direction: Direction.DESC }; + this.pageLink = new PageLink(10, 0, null, sortOrder); + this.dataSource = new EntityVersionsDatasource(this.entitiesVersionControlService); + } + + ngOnInit() { + this.componentResize$ = new ResizeObserver(() => { + const showHidePageSize = this.elementRef.nativeElement.offsetWidth < hidePageSizePixelValue; + if (showHidePageSize !== this.hidePageSize) { + this.hidePageSize = showHidePageSize; + this.cd.markForCheck(); + } + }); + this.componentResize$.observe(this.elementRef.nativeElement); + } + + ngOnDestroy() { + if (this.componentResize$) { + this.componentResize$.disconnect(); + } + } + + branchChanged(newBranch: string) { + this.branch = newBranch; + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } + + ngAfterViewInit() { + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); + merge(this.sort.sortChange, this.paginator.page) + .pipe( + tap(() => this.updateData()) + ) + .subscribe(); + this.viewsInited = true; + if (!this.singleEntityMode) { + this.initFromDefaultBranch(); + } + } + + vcExport($event: Event) { + if ($event) { + $event.stopPropagation(); + } + } + + private initFromDefaultBranch() { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + } + + private updateData() { + this.pageLink.page = this.paginator.pageIndex; + this.pageLink.pageSize = this.paginator.pageSize; + this.pageLink.sortOrder.property = this.sort.active; + this.pageLink.sortOrder.direction = Direction[this.sort.direction.toUpperCase()]; + this.dataSource.loadEntityVersions(this.singleEntityMode, this.branch, this.externalEntityIdValue, this.pageLink); + } + + private resetSortAndFilter(update: boolean) { + this.branch = null; + this.pageLink.textSearch = null; + if (this.viewsInited) { + this.paginator.pageIndex = 0; + const sortable = this.sort.sortables.get('timestamp'); + this.sort.active = sortable.id; + this.sort.direction = 'desc'; + if (update) { + this.initFromDefaultBranch(); + } + } + } +} + +class EntityVersionsDatasource implements DataSource { + + private entityVersionsSubject = new BehaviorSubject([]); + private pageDataSubject = new BehaviorSubject>(emptyPageData()); + + public pageData$ = this.pageDataSubject.asObservable(); + + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} + + connect(collectionViewer: CollectionViewer): Observable> { + return this.entityVersionsSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entityVersionsSubject.complete(); + this.pageDataSubject.complete(); + } + + loadEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + const result = new ReplaySubject>(); + this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entityVersionsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + } + ); + return result; + } + + fetchEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + if (!branch) { + return of(emptyPageData()); + } else { + if (singleEntityMode) { + if (externalEntityId) { + return this.entitiesVersionControlService.listEntityVersions(pageLink, branch, externalEntityId, {ignoreErrors: true}); + } else { + return of(emptyPageData()); + } + } else { + return this.entitiesVersionControlService.listVersions(pageLink, branch, {ignoreErrors: true}); + } + } + } + + isEmpty(): Observable { + return this.entityVersionsSubject.pipe( + map((entityVersions) => !entityVersions.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html similarity index 98% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html index b512bb96f5..b48162d06f 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.git-repository-settings diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss similarity index 95% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss index ede3570e68..d1d55faf19 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss @@ -14,6 +14,9 @@ * limitations under the License. */ :host { + mat-card.vc-settings { + margin: 8px; + } .fields-group { padding: 0 16px 8px; margin-bottom: 10px; diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts similarity index 82% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index c2626ec062..2970987be7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { AdminService } from '@core/http/admin.service'; import { @@ -30,13 +30,21 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { TranslateService } from '@ngx-translate/core'; import { isNotEmptyStr } from '@core/utils'; import { DialogService } from '@core/services/dialog.service'; +import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; +import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; @Component({ selector: 'tb-version-control-settings', templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] + styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] }) -export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class VersionControlSettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; versionControlSettingsForm: FormGroup; settings: EntitiesVersionControlSettings = null; @@ -62,7 +70,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On ngOnInit() { this.versionControlSettingsForm = this.fb.group({ repositoryUri: [null, [Validators.required]], - defaultBranch: [null, []], + defaultBranch: ['main', []], authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], username: [null, []], password: [null, []], @@ -77,16 +85,29 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { this.updateValidators(false); }); - this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + this.store.pipe( + select(selectHasVersionControl), + mergeMap((hasVersionControl) => { + if (hasVersionControl) { + return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( (settings) => { this.settings = settings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; + if (this.settings != null) { + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); }); } @@ -112,6 +133,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On } this.versionControlSettingsForm.reset(this.settings); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); } ); } @@ -131,18 +153,15 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.showChangePrivateKeyPassword = false; this.changePrivateKeyPassword = false; formDirective.resetForm(); - this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); } ); } }); } - confirmForm(): FormGroup { - return this.versionControlSettingsForm; - } - changePasswordChanged() { if (this.changePassword) { this.versionControlSettingsForm.get('password').patchValue(''); diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html new file mode 100644 index 0000000000..903460054b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -0,0 +1,25 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss new file mode 100644 index 0000000000..da8df4b469 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts new file mode 100644 index 0000000000..07bdbcba25 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -0,0 +1,61 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { FormGroup } from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-version-control', + templateUrl: './version-control.component.html', + styleUrls: ['./version-control.component.scss'] +}) +export class VersionControlComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent', {static: false}) versionControlSettingsComponent: VersionControlSettingsComponent; + + @Input() + detailsMode = false; + + @Input() + active = true; + + @Input() + singleEntityMode = false; + + @Input() + externalEntityId: EntityId; + + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + + constructor(private store: Store) { + + } + + ngOnInit() { + + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } + +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index d95b55e5f4..958e7b1c9b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -226,7 +226,7 @@ const routes: Routes = [ }, { path: 'vc', - component: VersionControlSettingsComponent, + component: VersionControlAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index aab8d4d56f..51612270df 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlSettingsComponent } from '@home/pages/admin/version-contr HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlSettingsComponent + VersionControlAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html new file mode 100644 index 0000000000..869067e1fb --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html @@ -0,0 +1,18 @@ + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts new file mode 100644 index 0000000000..a203f64426 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; + +@Component({ + selector: 'tb-version-control-admin-settings', + templateUrl: './version-control-admin-settings.component.html', + styleUrls: [] +}) +export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 3bec9b906c..33aac9ba44 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index 576ab1c9b2..feea05d76a 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -36,6 +36,7 @@ import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; +import { VcModule } from '@home/pages/vc/vc.module'; @NgModule({ exports: [ @@ -56,7 +57,8 @@ import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; AuditLogModule, ApiUsageModule, OtaUpdateModule, - UserModule + UserModule, + VcModule ], providers: [ { diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts new file mode 100644 index 0000000000..6054aaa14d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { ConfirmOnExitGuard } from '@core/guards/confirm-on-exit.guard'; +import { Authority } from '@shared/models/authority.enum'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; + +const routes: Routes = [ + { + path: 'vc', + component: VersionControlComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'version-control.version-control', + breadcrumb: { + label: 'version-control.version-control', + icon: 'history' + } + } + } +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) +export class VcRoutingModule { } diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts new file mode 100644 index 0000000000..4944356a9d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts @@ -0,0 +1,31 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { VcRoutingModule } from '@home/pages/vc/vc-routing.module'; + +@NgModule({ + declarations: [ + ], + imports: [ + CommonModule, + SharedModule, + VcRoutingModule + ] +}) +export class VcModule { } diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index 3a37d707ea..b189d70199 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - + {{ 'version-control.branch' | translate }} - + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts index 421710181a..e72235d89e 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -14,7 +14,17 @@ /// limitations under the License. /// -import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + forwardRef, + Input, + NgZone, + OnInit, + ViewChild +} from '@angular/core'; import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; import { Observable, of } from 'rxjs'; import { @@ -24,6 +34,7 @@ import { map, publishReplay, refCount, + share, switchMap, tap } from 'rxjs/operators'; @@ -32,6 +43,7 @@ import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { BranchInfo } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { isNotEmptyStr } from '@core/utils'; @Component({ selector: 'tb-branch-autocomplete', @@ -60,27 +72,48 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.requiredValue = coerceBooleanProperty(value); } + private disabledValue: boolean; + + get disabled(): boolean { + return this.disabledValue; + } + @Input() - disabled: boolean; + set disabled(value: boolean) { + this.disabledValue = coerceBooleanProperty(value); + if (this.disabledValue) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } @Input() selectDefaultBranch = true; + @Input() + selectionMode = false; + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; - filteredBranches: Observable>; + filteredBranches: Observable>; - branches: Observable>; + branches: Observable> = null; + + defaultBranch: BranchInfo = null; searchText = ''; private dirty = false; + private ignoreClosedPanel = false; + private propagateChange = (v: any) => { }; constructor(private store: Store, private entitiesVersionControlService: EntitiesVersionControlService, - private fb: FormBuilder) { + private fb: FormBuilder, + private zone: NgZone) { this.branchFormGroup = this.fb.group({ branch: [null, []] }); @@ -94,17 +127,36 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } ngOnInit() { - - this.branches = null; this.filteredBranches = this.branchFormGroup.get('branch').valueChanges .pipe( + tap((value: BranchInfo | string) => { + let modelValue: BranchInfo | null; + if (typeof value === 'string' || !value) { + if (!this.selectionMode && typeof value === 'string' && isNotEmptyStr(value)) { + modelValue = {name: value, default: false}; + } else { + modelValue = null; + } + } else { + modelValue = value; + } + this.updateView(modelValue); + }), + map(value => { + if (value) { + if (typeof value === 'string') { + return value; + } else { + return value.name; + } + } else { + return ''; + } + }), debounceTime(150), distinctUntilChanged(), - tap(value => { - this.updateView(value); - }), - map(value => value ? value : ''), - switchMap(branch => this.fetchBranches(branch)) + switchMap(name => this.fetchBranches(name)), + share() ); } @@ -113,24 +165,16 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; - if (this.disabled) { - this.branchFormGroup.disable({emitEvent: false}); - } else { - this.branchFormGroup.enable({emitEvent: false}); - } } - selectDefaultBranchIfNeeded(): void { - if (this.selectDefaultBranch && !this.modelValue) { - this.getBranches().subscribe( + selectDefaultBranchIfNeeded(ignoreLoading = true, force = false): void { + if ((this.selectDefaultBranch && !this.modelValue) || force) { + this.getBranches(ignoreLoading).subscribe( (data) => { - if (data && data.length) { - const defaultBranch = data.find(branch => branch.default); - if (defaultBranch) { - this.modelValue = defaultBranch.name; - this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); - this.propagateChange(this.modelValue); - } + if (this.defaultBranch || force) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); } } ); @@ -141,9 +185,9 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.searchText = ''; this.modelValue = value; if (value != null) { - this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue({name: value}, {emitEvent: false}); } else { - this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: false}); this.selectDefaultBranchIfNeeded(); } this.dirty = true; @@ -156,31 +200,53 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } } - updateView(value: string | null) { - if (this.modelValue !== value) { - this.modelValue = value; + onPanelClosed() { + if (this.ignoreClosedPanel) { + this.ignoreClosedPanel = false; + } else { + if (this.selectionMode && !this.branchFormGroup.get('branch').value && this.defaultBranch) { + this.zone.run(() => { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: true}); + }, 0); + } + } + } + + updateView(value: BranchInfo | null) { + if (this.modelValue !== value?.name) { + this.modelValue = value?.name; this.propagateChange(this.modelValue); } } - displayBranchFn(branch?: string): string | undefined { - return branch ? branch : undefined; + displayBranchFn(branch?: BranchInfo): string | undefined { + return branch ? branch.name : undefined; } - fetchBranches(searchText?: string): Observable> { + fetchBranches(searchText?: string): Observable> { this.searchText = searchText; return this.getBranches().pipe( - map(branches => branches.map(branch => branch.name).filter(branchName => { - return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; - })) + map(branches => { + let res = branches.filter(branch => { + return searchText ? branch.name.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (!this.selectionMode && isNotEmptyStr(searchText) && !res.find(b => b.name === searchText)) { + res = [{name: searchText, default: false}, ...res]; + } + return res; + } + ) ); } - getBranches(): Observable> { + getBranches(ignoreLoading = true): Observable> { if (!this.branches) { - const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading, ignoreErrors: true}); this.branches = branchesObservable.pipe( catchError(() => of([] as Array)), + tap((data) => { + this.defaultBranch = data.find(branch => branch.default); + }), publishReplay(1), refCount() ); @@ -189,6 +255,7 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } clear() { + this.ignoreClosedPanel = true; this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); setTimeout(() => { this.branchInput.nativeElement.blur(); diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index d38aee4d96..115eea4610 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { AssetId } from './id/asset-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; -export interface Asset extends BaseData { +export interface Asset extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/base-data.ts b/ui-ngx/src/app/shared/models/base-data.ts index 3af9bb9059..7b3ea9a642 100644 --- a/ui-ngx/src/app/shared/models/base-data.ts +++ b/ui-ngx/src/app/shared/models/base-data.ts @@ -27,6 +27,12 @@ export interface BaseData { label?: string; } +export interface ExportableEntity { + createdTime?: number; + id?: T; + externalId?: T; +} + export function hasIdEquals(id1: HasId, id2: HasId): boolean { if (isDefinedAndNotNull(id1) && isDefinedAndNotNull(id2)) { return id1.id === id2.id; diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 435edf263f..67c1429189 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -17,8 +17,9 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; import { TenantId } from './id/tenant-id'; +import { ExportableEntity } from '@shared/models/base-data'; -export interface Customer extends ContactBased { +export interface Customer extends ContactBased, ExportableEntity { tenantId: TenantId; title: string; additionalInfo?: any; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 4332e484f6..12e1c83ada 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DashboardId } from '@shared/models/id/dashboard-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; @@ -23,7 +23,7 @@ import { Timewindow } from '@shared/models/time/time.models'; import { EntityAliases } from './alias.models'; import { Filters } from '@shared/models/query/query.models'; -export interface DashboardInfo extends BaseData { +export interface DashboardInfo extends BaseData, ExportableEntity { tenantId?: TenantId; title?: string; image?: string; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index f5fc25e69d..c4c29f5da6 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DeviceId } from './id/device-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; @@ -560,7 +560,7 @@ export interface DeviceProfileData { provisionConfiguration?: DeviceProvisionConfiguration; } -export interface DeviceProfile extends BaseData { +export interface DeviceProfile extends BaseData, ExportableEntity { tenantId?: TenantId; name: string; description?: string; @@ -685,7 +685,7 @@ export interface DeviceData { transportConfiguration: DeviceTransportConfiguration; } -export interface Device extends BaseData { +export interface Device extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/rule-chain.models.ts b/ui-ngx/src/app/shared/models/rule-chain.models.ts index 85d6047191..01f9d268c3 100644 --- a/ui-ngx/src/app/shared/models/rule-chain.models.ts +++ b/ui-ngx/src/app/shared/models/rule-chain.models.ts @@ -14,14 +14,14 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainId } from '@shared/models/id/rule-chain-id'; import { RuleNodeId } from '@shared/models/id/rule-node-id'; import { RuleNode, RuleNodeComponentDescriptor, RuleNodeType } from '@shared/models/rule-node.models'; import { ComponentType } from '@shared/models/component-descriptor.models'; -export interface RuleChain extends BaseData { +export interface RuleChain extends BaseData, ExportableEntity { tenantId: TenantId; name: string; firstRuleNodeId: RuleNodeId; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0c890ff9b4..d574a1b55f 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -43,6 +43,7 @@ export interface BranchInfo { } export interface EntityVersion { + timestamp: number; id: string; name: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c37ba685e0..d5bac68d48 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3109,6 +3109,8 @@ "json-value-required": "JSON value is required." }, "version-control": { + "version-control": "Version control", + "management": "Version control management", "branch": "Branch", "select-branch": "Select branch", "branch-required": "Branch is required", @@ -3118,7 +3120,13 @@ "version-name-required": "Version name is required", "export-entity-relations": "Export entity relations", "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", - "export-to-git": "Export to Git" + "export-to-git": "Export to Git", + "entity-versions": "Entity versions", + "versions": "Versions", + "created-time": "Created time", + "version-id": "Version ID", + "no-entity-versions-text": "No entity versions found", + "no-versions-text": "No versions found" }, "widget": { "widget-library": "Widgets Library", From 4f2e1a25fdd772ed27dba9509dbec639dc46bd60 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 12:58:55 +0300 Subject: [PATCH 114/262] Docker compose scripts for TB Version Control Executor --- .../vc/EntitiesVersionControlService.java | 2 - .../queue/discovery/HashPartitionService.java | 2 +- .../DefaultClusterVersionControlService.java | 17 ++++--- docker/.env | 1 + docker/.gitignore | 1 + docker/docker-compose.confluent.yml | 6 +++ docker/docker-compose.kafka.yml | 10 ++++ docker/docker-compose.postgres.volumes.yml | 10 ++++ docker/docker-compose.pubsub.yml | 33 +++--------- docker/docker-compose.rabbitmq.yml | 32 +++--------- docker/docker-compose.service-bus.yml | 30 +++-------- docker/docker-compose.yml | 32 ++++++++++++ docker/docker-create-log-folders.sh | 2 + docker/tb-vc-executor.env | 2 + docker/tb-vc-executor/conf/logback.xml | 51 +++++++++++++++++++ .../tb-vc-executor/conf/tb-vc-executor.conf | 23 +++++++++ .../server/msa/ThingsBoardDbInstaller.java | 10 +++- msa/vc-executor-docker/docker/Dockerfile | 3 ++ 18 files changed, 181 insertions(+), 86 deletions(-) create mode 100644 docker/tb-vc-executor.env create mode 100644 docker/tb-vc-executor/conf/logback.xml create mode 100644 docker/tb-vc-executor/conf/tb-vc-executor.conf diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 566e093cb4..08b92db96a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -50,10 +50,8 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index e2325be878..583e196e2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,7 +54,7 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; - @Value("${queue.vc.topic}") + @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") private Integer vcPartitions; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..5fcb9985bd 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -188,8 +188,9 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); long startTs = System.currentTimeMillis(); - log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); - ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + log.trace("[{}][{}] RECEIVED task: {}", ctx.getTenantId(), ctx.getRequestId(), msg); + int threadIdx = Math.abs(ctx.getTenantId().hashCode() % ioPoolSize); + ListenableFuture future = ioThreads.get(threadIdx).submit(() -> processMessage(ctx, msg)); logTaskExecution(ctx, future, startTs); futures.add(future); } @@ -435,13 +436,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); if (e.isPresent()) { + log.debug("[{}][{}] Failed to process task", ctx.getTenantId(), ctx.getRequestId(), e.get()); builder.setError(e.get().getMessage()); - } - if (enrichFunction != null) { - builder = enrichFunction.apply(builder); } else { - builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + if (enrichFunction != null) { + builder = enrichFunction.apply(builder); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + log.debug("[{}][{}] Processed task", ctx.getTenantId(), ctx.getRequestId()); } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); diff --git a/docker/.env b/docker/.env index 11e44fdde8..af0f537603 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport +TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor TB_VERSION=latest diff --git a/docker/.gitignore b/docker/.gitignore index eee422d573..9c4c778f28 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -5,4 +5,5 @@ tb-node/db/** tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log +tb-vc-executor/log/** !.env diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 6983dd3d3e..3d5abd0abe 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -61,3 +61,9 @@ services: tb-snmp-transport: env_file: - queue-confluent.env + tb-vc-executor1: + env_file: + - queue-confluent.env + tb-vc-executor2: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 2184528e53..09c4554562 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -90,3 +90,13 @@ services: - queue-kafka.env depends_on: - kafka + tb-vc-executor1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-vc-executor2: + env_file: + - queue-kafka.env + depends_on: + - kafka \ No newline at end of file diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 704e9400ae..019e087c48 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -53,6 +53,13 @@ services: tb-snmp-transport: volumes: - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + volumes: postgres-db-volume: @@ -76,3 +83,6 @@ volumes: tb-snmp-transport-log-volume: external: true name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 0364957ee6..c03132d730 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -23,59 +23,40 @@ services: tb-core1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-pubsub.env + tb-vc-executor2: + env_file: + - queue-pubsub.env + diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index 1eb37709e5..d1acc32014 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-rabbitmq.env + tb-vc-executor2: + env_file: + - queue-rabbitmq.env \ No newline at end of file diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index c511658b31..6e39de0baa 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -23,57 +23,39 @@ services: tb-core1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-service-bus.env tb-lwm2m-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-service-bus.env + tb-vc-executor2: + env_file: + - queue-service-bus.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82fbaeed74..1ba6eda32d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -257,6 +257,38 @@ services: - "8080" env_file: - tb-web-ui.env + tb-vc-executor1: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor1 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 + tb-vc-executor2: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor2 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 5af0f96377..ba945a19df 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -26,3 +26,5 @@ mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log + +mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log diff --git a/docker/tb-vc-executor.env b/docker/tb-vc-executor.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-vc-executor.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml new file mode 100644 index 0000000000..dc95b3a885 --- /dev/null +++ b/docker/tb-vc-executor/conf/logback.xml @@ -0,0 +1,51 @@ + + + + + + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.log + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/docker/tb-vc-executor/conf/tb-vc-executor.conf b/docker/tb-vc-executor/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..f140e3fb76 --- /dev/null +++ b/docker/tb-vc-executor/conf/tb-vc-executor.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-vc-executor/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-vc-executor/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-vc-executor.out +export LOADER_PATH=/usr/share/tb-vc-executor/conf diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 417799f64f..de55a3afa3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -34,6 +34,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; + private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; private final DockerComposeExecutor dockerCompose; @@ -44,6 +45,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; + private final String tbVcExecutorLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -61,6 +63,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; + tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -72,6 +75,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); + env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); dockerCompose.withEnv(env); } @@ -104,6 +108,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); + dockerCompose.invokeDocker(); + dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); @@ -126,10 +133,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); + copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); } diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile index c2f9640a6b..f4ff25e0e5 100644 --- a/msa/vc-executor-docker/docker/Dockerfile +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -18,6 +18,9 @@ FROM thingsboard/openjdk11 COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ +RUN mkdir -p /home/thingsboard/.config/jgit +RUN chown -R ${pkg.user}:${pkg.user} /home/thingsboard + RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-vc-executor.sh /usr/bin From 5bb9bdface4f9b03c2c9690c6c7e478b32bc05b5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 13:43:35 +0300 Subject: [PATCH 115/262] Cache for VersionControlSettings --- .../service/session/SessionRedisCache.java | 2 +- .../DefaultEntitiesVersionControlService.java | 66 ++----------- ...efaultTbVersionControlSettingsService.java | 98 +++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 2 - .../vc/TbVersionControlSettingsService.java | 31 ++++++ .../VersionControlSettingsCaffeineCache.java | 36 +++++++ .../vc/VersionControlSettingsRedisCache.java | 41 ++++++++ .../src/main/resources/thingsboard.yml | 3 + .../server/common/data/CacheConstants.java | 1 + 9 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java index cd3e8cb56b..dc12326bdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java @@ -33,7 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; public class SessionRedisCache extends RedisTbTransactionalCache { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7ad3e309e0..7f791163a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,9 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -36,7 +34,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -46,7 +43,6 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -61,7 +57,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -86,10 +81,10 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final TbVersionControlSettingsService vcSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AdminSettingsService adminSettingsService; private final TransactionTemplate transactionTemplate; private ListeningExecutorService executor; @@ -311,54 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; + return vcSettingsService.get(tenantId); } @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - EntitiesVersionControlSettings storedSettings = null; - if (adminSettings != null) { - try { - storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } + versionControlSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); try { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.initRepository(tenantId, versionControlSettings).get(); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - return savedVersionControlSettings; + return vcSettingsService.save(tenantId, versionControlSettings); } @Override public void deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + if (vcSettingsService.delete(tenantId)) { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.clearRepository(tenantId).get(); } @@ -366,8 +331,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); - settings = this.restoreCredentials(settings, storedSettings); + settings = this.vcSettingsService.restore(tenantId, settings); try { //TODO: ashvayka: replace future.get with deferred result. gitServiceQueue.testRepository(tenantId, settings).get(); @@ -379,7 +343,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private String getCauseMessage(Exception e) { String message; - if(e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())){ + if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { message = e.getCause().getMessage(); } else { message = e.getMessage(); @@ -387,20 +351,4 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return message; } - private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - if (storedSettings != null) { - settings.setPassword(storedSettings.getPassword()); - } - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - if (storedSettings != null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java new file mode 100644 index 0000000000..a62b046ee2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + + @Override + public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { + EntitiesVersionControlSettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public EntitiesVersionControlSettings get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + }, true); + } + + @Override + public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedVersionControlSettings; + } + + @Override + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 08b92db96a..1d8cfca00c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -36,8 +36,6 @@ import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { - String SETTINGS_KEY = "entitiesVersionControl"; - ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java new file mode 100644 index 0000000000..178499c6c7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; + +public interface TbVersionControlSettingsService { + + EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + EntitiesVersionControlSettings get(TenantId tenantId); + + EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java new file mode 100644 index 0000000000..8cab03ca29 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("VersionControlCache") +public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java new file mode 100644 index 0000000000..9c778d6e4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("VersionControlCache") +public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { + + public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index aaec0852ca..62c8052254 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,6 +433,9 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + vcSettings: + timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 571af34086..85ea116f0a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,4 +31,5 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; + public static final String VC_SETTINGS_CACHE = "vcSettings"; } From ed44304eac4f779251912faf6a34349003f691f4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 16:26:39 +0300 Subject: [PATCH 116/262] UI: Branch select improvements. Entity versions table improvements. --- .../DefaultGitVersionControlQueueService.java | 9 ++ common/cluster-api/src/main/proto/queue.proto | 2 + .../DefaultClusterVersionControlService.java | 11 +- .../server/service/sync/vc/GitRepository.java | 2 +- ui-ngx/src/app/core/http/admin.service.ts | 17 ++- .../http/entities-version-control.service.ts | 45 +++++- .../vc/entity-versions-table.component.html | 24 ++- .../vc/entity-versions-table.component.ts | 33 ++++- .../vc/branch-autocomplete.component.html | 8 +- .../vc/branch-autocomplete.component.ts | 140 ++++++++++-------- .../assets/locale/locale.constant-en_US.json | 3 +- 11 files changed, 210 insertions(+), 84 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..66e0442297 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -150,6 +150,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -159,6 +162,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch).setEntityType(entityType.name()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -171,6 +177,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..82070a5895 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -724,6 +724,8 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; int32 pageSize = 5; int32 page = 6; + string sortProperty = 7; + string sortDirection = 8; } message EntityVersionProto { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..a27a4862bf 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -290,7 +291,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } else { path = null; } - var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage())); + SortOrder sortOrder = null; + if (StringUtils.isNotEmpty(request.getSortProperty())) { + var direction = SortOrder.Direction.DESC; + if (StringUtils.isNotEmpty(request.getSortDirection())) { + direction = SortOrder.Direction.valueOf(request.getSortDirection()); + } + sortOrder = new SortOrder(request.getSortProperty(), direction); + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage(), null, sortOrder)); reply(ctx, Optional.empty(), builder -> builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() .setTotalPages(data.getTotalPages()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..a1a46d5b25 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -281,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 79bb6580b8..7c12bd3a5d 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -26,6 +26,8 @@ import { TestSmsRequest, UpdateMessage } from '@shared/models/settings.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' @@ -33,7 +35,8 @@ import { export class AdminService { constructor( - private http: HttpClient + private http: HttpClient, + private entitiesVersionControlService: EntitiesVersionControlService ) { } public getAdminSettings(key: string, config?: RequestConfig): Observable> { @@ -72,11 +75,19 @@ export class AdminService { public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, config?: RequestConfig): Observable { return this.http.post('/api/admin/vcSettings', versionControlSettings, - defaultHttpOptionsFromConfig(config)); + defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index f8668f2ecf..eba229de67 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,30 +17,63 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { Observable } from 'rxjs'; +import { combineLatest, Observable, of } from 'rxjs'; import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; -import { DeviceInfo } from '@shared/models/device.models'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { createSelector, select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, combineAll, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' }) export class EntitiesVersionControlService { + branchList: Array = null; + constructor( - private http: HttpClient + private http: HttpClient, + private store: Store ) { + + this.store.pipe(select(selectIsUserLoaded)).subscribe( + () => { + this.branchList = null; + } + ); + } + + public clearBranchList(): void { + this.branchList = null; } - public listBranches(config?: RequestConfig): Observable> { - return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + public listBranches(): Observable> { + if (!this.branchList) { + return this.http.get>('/api/entities/vc/branches', + defaultHttpOptionsFromConfig({ignoreErrors: true, ignoreLoading: false})).pipe( + catchError(() => of([] as Array)), + tap((list) => { + this.branchList = list; + }) + ); + } else { + return of(this.branchList); + } } public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { - return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + const branch = request.branch; + if (this.branchList && !this.branchList.find(b => b.name === branch)) { + this.branchList = null; + } + }) + ); } public listEntityVersions(pageLink: PageLink, branch: string, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b47c2fcc79..f5aa4a04d2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -48,27 +48,41 @@ - {{ 'version-control.version-id' | translate }} + {{ 'version-control.version-id' | translate }} - {{ entityVersion.id }} + + + - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} - + - {{ singleEntityMode ? 'version-control.no-entity-versions-text' : 'version-control.no-versions-text' }} + {{ 'common.loading' | translate }}
7) { + versionId = versionId.slice(0, 7); + } + return versionId; + } + private initFromDefaultBranch() { - this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } else { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(true); + } } private updateData() { @@ -164,7 +182,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { - this.branch = null; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; @@ -185,6 +202,8 @@ class EntityVersionsDatasource implements DataSource { public pageData$ = this.pageDataSubject.asObservable(); + public dataLoading = true; + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} connect(collectionViewer: CollectionViewer): Observable> { @@ -199,6 +218,7 @@ class EntityVersionsDatasource implements DataSource { loadEntityVersions(singleEntityMode: boolean, branch: string, externalEntityId: EntityId, pageLink: PageLink): Observable> { + this.dataLoading = true; const result = new ReplaySubject>(); this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( catchError(() => of(emptyPageData())), @@ -207,6 +227,7 @@ class EntityVersionsDatasource implements DataSource { this.entityVersionsSubject.next(pageData.data); this.pageDataSubject.next(pageData); result.next(pageData); + this.dataLoading = false; } ); return result; diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index b189d70199..74672b1772 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -17,12 +17,14 @@ --> {{ 'version-control.branch' | translate }} - + [matAutocomplete]="branchAutocomplete"> - + +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts new file mode 100644 index 0000000000..368a4bf441 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -0,0 +1,84 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-entity-version-export', + templateUrl: './entity-version-export.component.html', + styleUrls: [] +}) +export class EntityVersionExportComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + entityId: EntityId; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + exportFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.exportFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null, null); + } + } + + export(): void { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result, request.branch); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index f5aa4a04d2..11d8b2a2e2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -31,10 +31,12 @@ - diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index 9017de60b8..ec5cb02256 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -78,7 +78,7 @@ tb-branch-autocomplete { mat-form-field { font-size: 16px; - width: 200px; + width: 250px; .mat-form-field-wrapper { padding-bottom: 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 884bab417b..436267fe2f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -21,8 +21,8 @@ import { ElementRef, Input, OnDestroy, - OnInit, - ViewChild + OnInit, Renderer2, + ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -33,7 +33,7 @@ import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, map, tap } from 'rxjs/operators'; -import { EntityVersion } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -42,6 +42,10 @@ import { hidePageSizePixelValue } from '@shared/models/constants'; import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; import { isNotEmptyStr } from '@core/utils'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -68,6 +72,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; + vcExportPopover: TbPopoverComponent; + private componentResize$: ResizeObserver; @Input() @@ -94,12 +100,18 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + @Input() + entityId: EntityId; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private popoverService: TbPopoverService, + private renderer: Renderer2, private cd: ChangeDetectorRef, + private viewContainerRef: ViewContainerRef, private elementRef: ElementRef) { super(store); this.dirtyValue = !this.activeValue; @@ -148,10 +160,36 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } - vcExport($event: Event) { + toggleVcExport($event: Event, exportButton: MatButton) { if ($event) { $event.stopPropagation(); } + const trigger = exportButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + { + branch: this.branch, + entityId: this.entityId, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + this.vcExportPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { + if (!visible) { + this.vcExportPopover = null; + } + }); + } } versionIdContent(entityVersion: EntityVersion): string { diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html deleted file mode 100644 index e7ea79cd73..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html +++ /dev/null @@ -1,77 +0,0 @@ - -
- -

{{ (createResult ? 'version-control.entity-version-exported' : 'version-control.export-entity-version') | translate }}

- - - - - -
-
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
-
-
-
-
- - -
-
- -
-
diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts deleted file mode 100644 index 9d3affea42..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts +++ /dev/null @@ -1,105 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; -import { ErrorStateMatcher } from '@angular/material/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; -import { Router } from '@angular/router'; -import { DialogComponent } from '@app/shared/components/dialog.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - SingleEntityVersionCreateRequest, - VersionCreateRequestType, - VersionCreationResult -} from '@shared/models/vc.models'; -import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; -import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; -import { TranslateService } from '@ngx-translate/core'; - -export interface VcEntityExportDialogData { - entityId: EntityId; -} - -@Component({ - selector: 'tb-vc-entity-export-dialog', - templateUrl: './vc-entity-export-dialog.component.html', - providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], - styleUrls: [] -}) -export class VcEntityExportDialogComponent extends DialogComponent - implements OnInit, ErrorStateMatcher { - - exportFormGroup: FormGroup; - - submitted = false; - - createResult: VersionCreationResult; - - createResultMessage: SafeHtml; - - constructor(protected store: Store, - protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, - @SkipSelf() private errorStateMatcher: ErrorStateMatcher, - public dialogRef: MatDialogRef, - private entitiesVersionControlService: EntitiesVersionControlService, - private translate: TranslateService, - private domSanitizer: DomSanitizer, - private fb: FormBuilder) { - super(store, router, dialogRef); - - this.exportFormGroup = this.fb.group({ - branch: [null, [Validators.required]], - versionName: [null, [Validators.required]], - saveRelations: [false, []] - }); - } - - ngOnInit(): void { - } - - isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { - const originalErrorState = this.errorStateMatcher.isErrorState(control, form); - const customErrorState = !!(control && control.invalid && this.submitted); - return originalErrorState || customErrorState; - } - - cancel(): void { - this.dialogRef.close(); - } - - export(): void { - this.submitted = true; - const request: SingleEntityVersionCreateRequest = { - entityId: this.data.entityId, - branch: this.exportFormGroup.get('branch').value, - versionName: this.exportFormGroup.get('versionName').value, - config: { - saveRelations: this.exportFormGroup.get('saveRelations').value - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - this.createResult = result; - const message = this.translate.instant('version-control.export-entity-version-result-message', - {name: result.version.name, commitId: result.version.id}); - this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); - }); - } -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 903460054b..c631819128 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -21,5 +21,6 @@ diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 07bdbcba25..2ca68255ae 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -44,6 +44,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() externalEntityId: EntityId; + @Input() + entityId: EntityId; + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 1730c69084..ff08af8039 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,11 +22,6 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - VcEntityExportDialogComponent, - VcEntityExportDialogData -} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -46,17 +41,6 @@ export class HomeDialogsService { } } - public exportVcEntity(entityId: EntityId): Observable { - return this.dialog.open(VcEntityExportDialogComponent, - { - disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - entityId - } - }).afterClosed(); - } - private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index e5accc15a9..3a3f2dda92 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 75171b4532..00ed882cd9 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,12 +46,6 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} - - - - - + + + + +
+ + +   + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 436267fe2f..b60665f2b4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -29,10 +29,10 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityId } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; -import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; -import { catchError, map, tap } from 'rxjs/operators'; +import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; @@ -61,6 +61,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni displayedColumns = ['timestamp', 'id', 'name']; pageLink: PageLink; + textSearchMode = false; dataSource: EntityVersionsDatasource; hidePageSize = false; @@ -103,6 +104,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @ViewChild('searchInput') searchInputField: ElementRef; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @@ -148,6 +151,17 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } ngAfterViewInit() { + fromEvent(this.searchInputField.nativeElement, 'keyup') + .pipe( + debounceTime(400), + distinctUntilChanged(), + tap(() => { + this.paginator.pageIndex = 0; + this.updateData(); + }) + ) + .subscribe(); + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); merge(this.sort.sortChange, this.paginator.page) .pipe( @@ -155,7 +169,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni ) .subscribe(); this.viewsInited = true; - if (!this.singleEntityMode) { + if (!this.singleEntityMode || (this.activeValue && this.externalEntityIdValue)) { this.initFromDefaultBranch(); } } @@ -200,6 +214,22 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni return versionId; } + enterFilterMode() { + this.textSearchMode = true; + this.pageLink.textSearch = ''; + setTimeout(() => { + this.searchInputField.nativeElement.focus(); + this.searchInputField.nativeElement.setSelectionRange(0, 0); + }, 10); + } + + exitFilterMode() { + this.textSearchMode = false; + this.pageLink.textSearch = null; + this.paginator.pageIndex = 0; + this.updateData(); + } + private initFromDefaultBranch() { if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { this.paginator.pageIndex = 0; @@ -220,6 +250,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { + this.textSearchMode = false; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index 2970987be7..3462fbb728 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -33,7 +33,7 @@ import { DialogService } from '@core/services/dialog.service'; import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap } from 'rxjs/operators'; +import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; @Component({ @@ -87,6 +87,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On }); this.store.pipe( select(selectHasVersionControl), + take(1), mergeMap((hasVersionControl) => { if (hasVersionControl) { return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( From 905b60da6a4a6d006cfc86b30861358cb2efd190 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 12:32:51 +0300 Subject: [PATCH 121/262] Fix GitRepository.status command, Increased timeout between poll commands. --- application/src/main/resources/thingsboard.yml | 3 +++ .../thingsboard/server/service/sync/vc/GitRepository.java | 5 ++++- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 62c8052254..31e5d7a76e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -931,6 +931,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index a0c905852b..c22e18c004 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -241,7 +241,10 @@ public class GitRepository { public Status status() throws GitAPIException { org.eclipse.jgit.api.Status status = execute(git.status()); - return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + Set modified = new HashSet<>(); + modified.addAll(status.getModified()); + modified.addAll(status.getChanged()); + return new Status(status.getAdded(), modified, status.getRemoved()); } public Commit commit(String message) throws GitAPIException { diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 8c34a91e80..4e430e5ba6 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,6 +73,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" From 46e0c7c90897db26fa30c604bfd7dc6b4aa5a2ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 13:50:50 +0300 Subject: [PATCH 122/262] UI: Implement entity version restore --- .../DefaultEntitiesVersionControlService.java | 2 +- .../http/entities-version-control.service.ts | 12 ++- .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-export.component.html | 95 +++++++++++-------- .../vc/entity-version-export.component.scss | 21 ++++ .../vc/entity-version-export.component.ts | 16 +++- .../vc/entity-version-restore.component.html | 49 ++++++++++ .../vc/entity-version-restore.component.ts | 86 +++++++++++++++++ .../vc/entity-versions-table.component.html | 15 +++ .../vc/entity-versions-table.component.ts | 56 ++++++++--- .../vc/version-control.component.html | 3 +- .../vc/version-control.component.ts | 5 +- .../pages/asset/asset-tabs.component.html | 1 + .../customer/customer-tabs.component.html | 1 + .../dashboard/dashboard-tabs.component.html | 1 + .../device-profile-tabs.component.html | 3 +- .../pages/device/device-tabs.component.html | 1 + .../rulechain/rulechain-tabs.component.html | 1 + ui-ngx/src/app/shared/models/vc.models.ts | 29 ++++++ .../assets/locale/locale.constant-en_US.json | 9 +- 20 files changed, 347 insertions(+), 66 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 825171493e..e812595ca4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -190,7 +190,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - Futures.transform(future, entityData -> { + return Futures.transform(future, entityData -> { EntityImportResult importResult = transactionTemplate.execute(status -> { try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index eba229de67..86674f0569 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,13 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { combineLatest, Observable, of } from 'rxjs'; -import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { + BranchInfo, + EntityVersion, + VersionCreateRequest, + VersionCreationResult, + VersionLoadRequest, VersionLoadResult +} from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -95,4 +101,8 @@ export class EntitiesVersionControlService { return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public loadEntitiesVersion(request: VersionLoadRequest, config?: RequestConfig): Observable> { + return this.http.post>('/api/entities/vc/entity', request, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 38cf85e72c..6134708176 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -157,6 +157,7 @@ import { VersionControlSettingsComponent } from '@home/components/vc/version-con import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @NgModule({ declarations: @@ -284,7 +285,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], imports: [ CommonModule, @@ -406,7 +408,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html index 5791790b37..3838ea111d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html @@ -16,45 +16,58 @@ -->
- -

{{ 'version-control.create-entity-version' | translate }}

- - - - -
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
- - -
+
+ +

{{ 'version-control.create-entity-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+ + +
+
+
+
{{ resultMessage }}
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss new file mode 100644 index 0000000000..46e55cf4d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .export-result-message { + padding: 48px 8px 8px; + text-align: center; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts index 368a4bf441..2aa3123858 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -26,11 +26,12 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; @Component({ selector: 'tb-entity-version-export', templateUrl: './entity-version-export.component.html', - styleUrls: [] + styleUrls: ['./entity-version-export.component.scss'] }) export class EntityVersionExportComponent extends PageComponent implements OnInit { @@ -43,10 +44,16 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; + @Input() + onContentUpdated: () => void; + exportFormGroup: FormGroup; + resultMessage: string; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, private fb: FormBuilder) { super(store); } @@ -76,7 +83,12 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni type: VersionCreateRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { this.onClose(result, request.branch); } }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html new file mode 100644 index 0000000000..cb71696b4e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -0,0 +1,49 @@ + +
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + +
+
+
+ + {{ 'version-control.load-entity-relations' | translate }} + +
+
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts new file mode 100644 index 0000000000..2cc615c011 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -0,0 +1,86 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { SingleEntityVersionLoadRequest, VersionLoadRequestType, VersionLoadResult } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-entity-version-restore', + templateUrl: './entity-version-restore.component.html', + styleUrls: [] +}) +export class EntityVersionRestoreComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: (result: Array | null) => void; + + restoreFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.restoreFormGroup = this.fb.group({ + loadRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + restore(): void { + const request: SingleEntityVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + config: { + loadRelations: this.restoreFormGroup.get('loadRelations').value + }, + type: VersionLoadRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 5acfe3b0ba..2e5302881e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -99,6 +99,21 @@ {{ entityVersion.name }} + + + + +
+ +
+ + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index b60665f2b4..06c778d278 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -18,10 +18,10 @@ import { AfterViewInit, ChangeDetectorRef, Component, - ElementRef, + ElementRef, EventEmitter, Input, OnDestroy, - OnInit, Renderer2, + OnInit, Output, Renderer2, ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; @@ -33,7 +33,7 @@ import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; -import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult, VersionLoadResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -46,6 +46,7 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { MatButton } from '@angular/material/button'; import { TbPopoverComponent } from '@shared/components/popover.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @Component({ selector: 'tb-entity-versions-table', @@ -59,7 +60,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name']; + displayedColumns = ['timestamp', 'id', 'name', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; @@ -73,8 +74,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; - vcExportPopover: TbPopoverComponent; - private componentResize$: ResizeObserver; @Input() @@ -104,6 +103,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + @ViewChild('searchInput') searchInputField: ElementRef; @ViewChild(MatPaginator) paginator: MatPaginator; @@ -182,13 +184,13 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { - this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + const vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'left', true, null, { branch: this.branch, entityId: this.entityId, onClose: (result: VersionCreationResult | null, branch: string | null) => { - this.vcExportPopover.hide(); + vcExportPopover.hide(); if (result) { if (this.branch !== branch) { this.branchChanged(branch); @@ -196,13 +198,39 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } + }, + onContentUpdated: () => { + vcExportPopover.updatePosition(); + setTimeout(() => { + vcExportPopover.updatePosition(); + }); + } + }, {}, {}, {}, false); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + externalEntityId: this.externalEntityIdValue, + onClose: (result: Array | null) => { + restoreVersionPopover.hide(); + if (result && result.length) { + this.versionRestored.emit(); + } } }, {}, {}, {}, false); - this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { - if (!visible) { - this.vcExportPopover = null; - } - }); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index c631819128..e5e650ac85 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -22,5 +22,6 @@ + [externalEntityId]="externalEntityId" + (versionRestored)="versionRestored.emit()"> diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 2ca68255ae..da6828c4a0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; @@ -47,6 +47,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 3a3f2dda92..c115a43ab6 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html index 7524fd59f7..634443a397 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html index 21b20bcb82..61a030ac16 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html @@ -22,5 +22,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index e713e70690..effac9a7c8 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -75,8 +75,9 @@ label="{{ 'audit-log.audit-logs' | translate }}"> - diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index d84306d029..78d494d8ae 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html index c60910b565..49a2d83372 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html @@ -55,5 +55,6 @@ diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index d574a1b55f..4975b8ea2b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -15,16 +15,26 @@ /// import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; export interface VersionCreateConfig { saveRelations: boolean; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionCreateRequest { versionName: string; branch: string; @@ -37,6 +47,18 @@ export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { type: VersionCreateRequestType.SINGLE_ENTITY; } +export interface VersionLoadRequest { + branch: string; + versionId: string; + type: VersionLoadRequestType; +} + +export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { + externalEntityId: EntityId; + config: VersionLoadConfig; + type: VersionLoadRequestType.SINGLE_ENTITY; +} + export interface BranchInfo { name: string; default: boolean; @@ -54,3 +76,10 @@ export interface VersionCreationResult { modified: number; removed: number; } + +export interface VersionLoadResult { + entityType: EntityType; + created: number; + updated: number; + deleted: number; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 202597ade1..0a2f39bcd4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -57,7 +57,8 @@ "download": "Download", "next-with-label": "Next: {{label}}", "read-more": "Read more", - "hide": "Hide" + "hide": "Hide", + "restore": "Restore" }, "aggregation": { "aggregation": "Aggregation", @@ -3126,7 +3127,11 @@ "no-entity-versions-text": "No entity versions found", "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", - "create-version": "Create version" + "create-version": "Create version", + "nothing-to-commit": "Nothing to commit", + "restore-version": "Restore version", + "restore-entity-from-version": "Restore entity from version '{{versionName}}'", + "load-entity-relations": "Load entity relations" }, "widget": { "widget-library": "Widgets Library", From 6558b0b24d2d4a4baae5edaefab987cd1f38fe96 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:27:22 +0300 Subject: [PATCH 123/262] UI: Improve restore action --- .../home/components/vc/entity-versions-table.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 06c778d278..fcaa12f352 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -27,7 +27,7 @@ import { import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { EntityId } from '@shared/models/id/entity-id'; +import { EntityId, entityIdEquals } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; @@ -91,7 +91,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() set externalEntityId(externalEntityId: EntityId) { - if (this.externalEntityIdValue !== externalEntityId) { + if (!entityIdEquals(this.externalEntityIdValue, externalEntityId)) { this.externalEntityIdValue = externalEntityId; this.resetSortAndFilter(this.activeValue); if (!this.activeValue) { From c46b5b6448e81bca42bd4a7afde261ad6f030ac4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:34:25 +0300 Subject: [PATCH 124/262] Fix message --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0a2f39bcd4..24cdff2735 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3128,7 +3128,7 @@ "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", "create-version": "Create version", - "nothing-to-commit": "Nothing to commit", + "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations" From e4ed8c508098849efdbef94834d7e31e3a3235e2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 26 May 2022 15:10:46 +0300 Subject: [PATCH 125/262] Api for comparing current entity data to specific version; refactoring --- .../EntitiesVersionControlController.java | 13 +++ .../DefaultEntitiesExportImportService.java | 10 +- .../DefaultEntitiesVersionControlService.java | 32 +++++- .../DefaultGitVersionControlQueueService.java | 83 ++++++++++---- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 6 + .../{ => data}/ClearRepositoryGitRequest.java | 2 +- .../sync/vc/{ => data}/CommitGitRequest.java | 2 +- .../sync/vc/data/ContentsDiffGitRequest.java | 33 ++++++ .../{ => data}/EntitiesContentGitRequest.java | 3 +- .../{ => data}/EntityContentGitRequest.java | 2 +- .../vc/{ => data}/ListBranchesGitRequest.java | 3 +- .../vc/{ => data}/ListEntitiesGitRequest.java | 3 +- .../vc/{ => data}/ListVersionsGitRequest.java | 6 +- .../sync/vc/{ => data}/PendingGitRequest.java | 2 +- .../sync/vc/data/VersionsDiffGitRequest.java | 38 ++++++ .../sync/vc/{ => data}/VoidGitRequest.java | 5 +- common/cluster-api/src/main/proto/queue.proto | 32 ++++++ .../common/data/sync/vc/EntityDataDiff.java | 28 +++++ .../data/sync/vc/EntityVersionsDiff.java | 34 ++++++ .../load/EntityTypeVersionLoadConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 - .../thingsboard/common/util/JacksonUtil.java | 13 ++- .../DefaultClusterVersionControlService.java | 39 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 21 ++-- .../server/service/sync/vc/GitRepository.java | 108 ++++++++++++------ .../service/sync/vc/GitRepositoryService.java | 5 + 27 files changed, 433 insertions(+), 97 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ClearRepositoryGitRequest.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/CommitGitRequest.java (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntitiesContentGitRequest.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntityContentGitRequest.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListBranchesGitRequest.java (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListEntitiesGitRequest.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListVersionsGitRequest.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/PendingGitRequest.java (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/VoidGitRequest.java (85%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bafc93c19f..793ab0503c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -225,6 +226,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") + public DeferredResult compareEntityDataToVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid, + @RequestParam String versionId) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } @ApiOperation(value = "", notes = "" + "SINGLE_ENTITY:" + NEW_LINE + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 38d3b00213..777e84f348 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -24,18 +24,18 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7f791163a0..73eaafe842 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -57,11 +60,13 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -75,6 +80,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.transform; +import static com.google.common.util.concurrent.Futures.transformAsync; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -109,7 +117,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - return Futures.transformAsync(pendingCommit, commit -> { + return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { @@ -146,7 +154,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont break; } } - return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + return transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); }, executor); } @@ -195,7 +203,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) + .findExistingByName(false) .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); @@ -298,6 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } + @Override + public ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception { + HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); + + EntityId externalId = ((ExportableEntity) entity).getExternalId(); + if (externalId == null) externalId = entityId; + + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(true) + .build()); + return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion), + JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { + return new EntityDataDiff(currentVersion, otherVersion, rawDiff); + }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + } @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..270dc5b38c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; @@ -24,6 +22,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -36,6 +35,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -54,12 +54,23 @@ import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest; +import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; +import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -74,7 +85,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final DefaultEntitiesVersionControlService entitiesVersionControlService; private final Map> pendingRequestMap = new HashMap<>(); - private final ObjectMapper jsonMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, @@ -101,13 +111,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson; - try { - entityDataJson = jsonMapper.writeValueAsString(entityData); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } + String entityDataJson = JacksonUtil.toPrettyString(entityData); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -176,10 +180,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg)); } @Override @@ -201,19 +202,32 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg)); } @Override public ListenableFuture> listBranches(TenantId tenantId) { ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); + } - registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + @Override + public ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) { + String path = entityType != null ? getRelativePath(entityType, externalId) : ""; + VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2); + return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder() + .setPath(request.getPath()) + .setVersionId1(request.getVersionId1()) + .setVersionId2(request.getVersionId2()) + .build())); + } - return request.getFuture(); + @Override + public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { + ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); + return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() + .setContent1(content1) + .setContent2(content2))); } @Override @@ -246,6 +260,14 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + private ListenableFuture sendRequest(PendingGitRequest request, Consumer enrichFunction) { + registerAndSend(request, builder -> { + enrichFunction.accept(builder); + return builder.build(); + }, wrap(request.getFuture())); + return request.getFuture(); + } + @Override @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { @@ -334,6 +356,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); ((EntitiesContentGitRequest) request).getFuture() .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + } else if (vcResponseMsg.hasVersionsDiffResponse()) { + TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); + List entityVersionsDiffList = diffResponse.getDiffList().stream() + .map(diff -> EntityVersionsDiff.builder() + .externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()), + new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB()))) + .entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ? + toData(diff.getEntityDataAtVersion1()) : null) + .entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ? + toData(diff.getEntityDataAtVersion2()) : null) + .rawDiff(diff.getRawDiff()) + .build()) + .collect(Collectors.toList()); + ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); + } else if (vcResponseMsg.hasContentsDiffResponse()) { + String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); + ((ContentsDiffGitRequest) request).getFuture().set(diff); } } } @@ -354,7 +393,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @SuppressWarnings("rawtypes") @SneakyThrows private EntityExportData toData(String data) { - return jsonMapper.readValue(data, EntityExportData.class); + return JacksonUtil.fromString(data, EntityExportData.class); } private static TbQueueCallback wrap(SettableFuture future) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 1d8cfca00c..cb5ec52b6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; -import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -50,6 +49,8 @@ public interface EntitiesVersionControlService { ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index 00899a94e2..b7eb368055 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import java.util.List; @@ -58,6 +60,10 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); + + ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java index 86472e7882..09245eb895 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java index 3c83a34cfe..a989439d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java new file mode 100644 index 0000000000..7c3fb7a600 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class ContentsDiffGitRequest extends PendingGitRequest { + + private final String content1; + private final String content2; + + public ContentsDiffGitRequest(TenantId tenantId, String content1, String content2) { + super(tenantId); + this.content1 = content1; + this.content2 = content2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java index 2ded932e44..ad653edd0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java index c9ce3c76af..6a1d60a065 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java index c8219641da..c045030dc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java index 0c7216ef49..3fc531735e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java index 0c646b424c..d40a589646 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java @@ -13,15 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; - -import java.util.List; public class ListVersionsGitRequest extends PendingGitRequest> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java index e63ba04f45..fbec600ed0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java new file mode 100644 index 0000000000..e73706609f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +@Getter +public class VersionsDiffGitRequest extends PendingGitRequest> { + + private final String path; + private final String versionId1; + private final String versionId2; + + public VersionsDiffGitRequest(TenantId tenantId, String path, String versionId1, String versionId2) { + super(tenantId); + this.path = path; + this.versionId1 = versionId1; + this.versionId2 = versionId2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java index d829960ab4..c48bfa7a03 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java @@ -13,12 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; - -import java.util.List; public class VoidGitRequest extends PendingGitRequest { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..5a61dfa46a 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -784,6 +784,34 @@ message EntitiesContentResponseMsg { repeated string data = 1; } +message VersionsDiffRequestMsg { + string path = 1; + string versionId1 = 2; + string versionId2 = 3; +} + +message VersionsDiffResponseMsg { + repeated EntityVersionsDiff diff = 1; +} + +message EntityVersionsDiff { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + string entityDataAtVersion1 = 4; + string entityDataAtVersion2 = 5; + string rawDiff = 6; +} + +message ContentsDiffRequestMsg { + string content1 = 1; + string content2 = 2; +} + +message ContentsDiffResponseMsg { + string diff = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -804,6 +832,8 @@ message ToVersionControlServiceMsg { ListBranchesRequestMsg listBranchesRequest = 13; EntityContentRequestMsg entityContentRequest = 14; EntitiesContentRequestMsg entitiesContentRequest = 15; + VersionsDiffRequestMsg versionsDiffRequest = 16; + ContentsDiffRequestMsg contentsDiffRequest = 17; } message VersionControlResponseMsg { @@ -817,6 +847,8 @@ message VersionControlResponseMsg { ListVersionsResponseMsg listVersionsResponse = 8; EntityContentResponseMsg entityContentResponse = 9; EntitiesContentResponseMsg entitiesContentResponse = 10; + VersionsDiffResponseMsg versionsDiffResponse = 11; + ContentsDiffResponseMsg contentsDiffResponse = 12; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java new file mode 100644 index 0000000000..800583b0b8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +public class EntityDataDiff { + private EntityExportData currentVersion; + private EntityExportData otherVersion; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java new file mode 100644 index 0000000000..2104472047 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionsDiff { + private EntityId externalId; + private EntityExportData entityDataAtVersion1; + private EntityExportData entityDataAtVersion2; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 837986f82d..ffb1f25d3c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -23,5 +23,6 @@ import lombok.EqualsAndHashCode; public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; + private boolean findExistingEntityByName; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 31b34dbcb7..8ed9c34694 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,6 +21,5 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; - private boolean findExistingEntityByName; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..b6e2b54479 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -19,12 +19,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -32,6 +31,8 @@ import java.util.Set; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -96,6 +97,14 @@ public class JacksonUtil { } } + public static String toPrettyString(Object o) { + try { + return PRETTY_JSON_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public static JsonNode toJsonNode(String value) { if (value == null || value.isEmpty()) { return null; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 5fcb9985bd..47ab9fc2de 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -31,6 +31,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -91,6 +92,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.service.sync.vc.DefaultGitRepositoryService.fromRelativePath; + @Slf4j @TbVersionControlComponent @Service @@ -237,17 +240,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe vcService.fetch(ctx.getTenantId()); handleListBranches(ctx, msg.getListBranchesRequest()); } else if (msg.hasListEntitiesRequest()) { - vcService.fetch(ctx.getTenantId()); handleListEntities(ctx, msg.getListEntitiesRequest()); } else if (msg.hasListVersionRequest()) { vcService.fetch(ctx.getTenantId()); handleListVersions(ctx, msg.getListVersionRequest()); } else if (msg.hasEntityContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntityContentRequest(ctx, msg.getEntityContentRequest()); } else if (msg.hasEntitiesContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } else if (msg.hasVersionsDiffRequest()) { + handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); + } else if (msg.hasContentsDiffRequest()) { + handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); } } } @@ -322,6 +326,31 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); } + private void handleVersionsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.VersionsDiffRequestMsg request) throws IOException { + List diffList = vcService.getVersionsDiffList(ctx.getTenantId(), request.getPath(), request.getVersionId1(), request.getVersionId2()).stream() + .map(diff -> { + EntityId entityId = fromRelativePath(diff.getFilePath()); + return TransportProtos.EntityVersionsDiff.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityDataAtVersion1(diff.getFileContentAtCommit1()) + .setEntityDataAtVersion2(diff.getFileContentAtCommit2()) + .setRawDiff(diff.getDiffStringValue()) + .build(); + }) + .collect(Collectors.toList()); + + reply(ctx, builder -> builder.setVersionsDiffResponse(TransportProtos.VersionsDiffResponseMsg.newBuilder() + .addAllDiff(diffList))); + } + + private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { + String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); + reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() + .setDiff(diff))); + } + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); @@ -430,6 +459,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, e, null); } + private void reply(VersionControlRequestCtx ctx, Function enrichFunction) { + reply(ctx, Optional.empty(), enrichFunction); + } + private void reply(VersionControlRequestCtx ctx, Optional e, Function enrichFunction) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); VersionControlResponseMsg.Builder builder = VersionControlResponseMsg.newBuilder() diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 45c5e563a6..a759ddbfa8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import javax.annotation.PostConstruct; import java.io.File; @@ -167,6 +168,18 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return repository.getFileContentAtCommit(relativePath, versionId); } + @Override + public List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getDiffList(versionId1, versionId2, path); + } + + @Override + public String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getContentsDiff(content1, content2); + } + @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); @@ -178,12 +191,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - private GitRepository checkRepository(TenantId tenantId) { return Optional.ofNullable(repositories.get(tenantId)) .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); @@ -251,7 +258,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } - private EntityId fromRelativePath(String path) { + public static EntityId fromRelativePath(String path) { EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..921630a4ba 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -31,6 +31,12 @@ import org.eclipse.jgit.api.LsRemoteCommand; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.EditList; +import org.eclipse.jgit.diff.HistogramDiff; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -45,6 +51,7 @@ import org.eclipse.jgit.transport.sshd.JGitKeyCache; import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; @@ -54,6 +61,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -207,7 +215,7 @@ public class GitRepository { RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { if (treeWalk == null) { - throw new IllegalArgumentException("Not found"); + throw new IllegalArgumentException("File not found"); } ObjectId blobId = treeWalk.getObjectId(0); try (ObjectReader objectReader = git.getRepository().newObjectReader()) { @@ -248,40 +256,65 @@ public class GitRepository { .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } + public String getContentsDiff(String content1, String content2) throws IOException { + RawText rawContent1 = new RawText(content1.getBytes()); + RawText rawContent2 = new RawText(content2.getBytes()); -// public List getCommitChanges(Commit commit) throws IOException, GitAPIException { -// RevCommit revCommit = resolveCommit(commit.getId()); -// if (revCommit.getParentCount() == 0) { -// return null; // just takes the first parent of a commit, but should find a parent in branch provided -// } -// return execute(git.diff() -// .setOldTree(prepareTreeParser(git.getRepository().parseCommit(revCommit.getParent(0)))) -// .setNewTree(prepareTreeParser(revCommit))).stream() -// .map(diffEntry -> new Diff(diffEntry.getChangeType().name(), diffEntry.getOldPath(), diffEntry.getNewPath())) -// .collect(Collectors.toList()); -// } -// -// -// private AbstractTreeIterator prepareTreeParser(RevCommit revCommit) throws IOException { -// // from the commit we can build the tree which allows us to construct the TreeParser -// //noinspection Duplicates -// org.eclipse.jgit.lib.Repository repository = git.getRepository(); -// try (RevWalk walk = new RevWalk(repository)) { -// RevTree tree = walk.parseTree(revCommit.getTree().getId()); -// -// CanonicalTreeParser treeParser = new CanonicalTreeParser(); -// try (ObjectReader reader = repository.newObjectReader()) { -// treeParser.reset(reader, tree.getId()); -// } -// -// walk.dispose(); -// -// return treeParser; -// } -// } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + + EditList edits = new EditList(); + edits.addAll(new HistogramDiff().diff(RawTextComparator.DEFAULT, rawContent1, rawContent2)); + diffFormatter.format(edits, rawContent1, rawContent2); + return out.toString(); + } + + public List getDiffList(String commit1, String commit2, String path) throws IOException { + ObjectReader reader = git.getRepository().newObjectReader(); + + CanonicalTreeParser tree1Iter = new CanonicalTreeParser(); + ObjectId tree1 = resolveCommit(commit1).getTree(); + tree1Iter.reset(reader, tree1); + + CanonicalTreeParser tree2Iter = new CanonicalTreeParser(); + ObjectId tree2 = resolveCommit(commit2).getTree(); + tree2Iter.reset(reader, tree2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + if (StringUtils.isNotEmpty(path)) { + diffFormatter.setPathFilter(PathFilter.create(path)); + } + + return diffFormatter.scan(tree1, tree2).stream() + .map(diffEntry -> { + Diff diff = new Diff(); + try { + out.reset(); + diffFormatter.format(diffEntry); + diff.setDiffStringValue(out.toString()); + diff.setFilePath(diffEntry.getChangeType() != DiffEntry.ChangeType.DELETE ? diffEntry.getNewPath() : diffEntry.getOldPath()); + diff.setChangeType(diffEntry.getChangeType()); + try { + diff.setFileContentAtCommit1(getFileContentAtCommit(diff.getFilePath(), commit1)); + } catch (IllegalArgumentException ignored) { + } + try { + diff.setFileContentAtCommit2(getFileContentAtCommit(diff.getFilePath(), commit2)); + } catch (IllegalArgumentException ignored) { + } + return diff; + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000L, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -309,7 +342,7 @@ public class GitRepository { return null; }; - private static PageData iterableToPageData (Iterable iterable, + private static PageData iterableToPageData(Iterable iterable, Function mapper, PageLink pageLink, Function> comparatorFunction) { @@ -406,4 +439,13 @@ public class GitRepository { private final Set removed; } + @Data + public static class Diff { + private String filePath; + private DiffEntry.ChangeType changeType; + private String fileContentAtCommit1; + private String fileContentAtCommit2; + private String diffStringValue; + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 9d57eee3ec..ec69e0c80e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import java.io.IOException; import java.util.List; @@ -60,5 +61,9 @@ public interface GitRepositoryService { String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException; + + String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException; + void fetch(TenantId tenantId) throws GitAPIException; } From 576c34b41ae13d150f18dcbe7703c32e3add083f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 15:28:45 +0300 Subject: [PATCH 126/262] Delete repository on Tenant Deletion --- .../server/service/entitiy/tenant/DefaultTbTenantService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 2a8fc1bf0a..22baed8366 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.queue.TbQueueService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.Collections; +import java.util.concurrent.TimeUnit; @Service @TbCoreComponent @@ -38,6 +40,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T private final InstallScripts installScripts; private final TbQueueService tbQueueService; private final TenantProfileService tenantProfileService; + private final EntitiesVersionControlService versionControlService; @Override public Tenant save(Tenant tenant) throws ThingsboardException { @@ -70,6 +73,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T tenantService.deleteTenant(tenantId); tenantProfileCache.evict(tenantId); notificationEntityService.notifyDeleteTenant(tenant); + versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES); } catch (Exception e) { throw handleException(e); } From e86b37b4afe464fcd39475bd82d5d2a9943011ff Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 27 May 2022 13:14:18 +0300 Subject: [PATCH 127/262] UI: Initial implementation of entity versions diff component --- ui-ngx/angular.json | 6 +- ui-ngx/package.json | 2 + .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-diff.component.html | 35 +++++++ .../vc/entity-version-diff.component.scss | 23 +++++ .../vc/entity-version-diff.component.ts | 92 +++++++++++++++++++ .../vc/entity-versions-table.component.html | 9 +- .../vc/entity-versions-table.component.ts | 23 +++++ .../src/app/shared/models/ace/ace.models.ts | 16 ++++ .../assets/locale/locale.constant-en_US.json | 4 +- ui-ngx/yarn.lock | 12 +++ 11 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index f4d4412723..da4d21a84b 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -79,7 +79,8 @@ "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css", "node_modules/@geoman-io/leaflet-geoman-free/dist/leaflet-geoman.css", "node_modules/prismjs/themes/prism.css", - "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css" + "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css", + "node_modules/ace-diff/dist/ace-diff.min.css" ], "stylePreprocessorOptions": { "includePaths": [ @@ -130,7 +131,8 @@ "jstree", "qrcode", "wcwidth", - "leaflet-polylinedecorator" + "leaflet-polylinedecorator", + "ace-diff" ] }, "configurations": { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index e54606dc08..fa21cc9525 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -41,6 +41,7 @@ "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", "ace-builds": "^1.4.13", + "ace-diff": "^3.0.3", "angular-gridster2": "~12.1.1", "angular2-hotkeys": "^2.4.0", "canvas-gauges": "^2.1.7", @@ -105,6 +106,7 @@ "@angular/compiler-cli": "^12.2.13", "@angular/language-service": "^12.2.13", "@ngtools/webpack": "~12.2.13", + "@types/ace-diff": "^2.1.1", "@types/canvas-gauges": "^2.1.4", "@types/flot": "^0.0.32", "@types/jasmine": "~3.10.2", diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 6134708176..9afe26d8f1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -158,6 +158,7 @@ import { VersionControlComponent } from '@home/components/vc/version-control.com import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; +import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; @NgModule({ declarations: @@ -286,7 +287,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], imports: [ CommonModule, @@ -409,7 +411,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html new file mode 100644 index 0000000000..f080fcaf99 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html @@ -0,0 +1,35 @@ + +
+ +

{{ 'version-control.diff-entity-with-version' | translate: {versionName} }}

+ + + + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss new file mode 100644 index 0000000000..5db7fd40e5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .diff-viewer { + position: relative; + border: 1px solid #c0c0c0; + margin-top: 16px; + margin-bottom: 16px; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts new file mode 100644 index 0000000000..f46375b152 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -0,0 +1,92 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { getAceDiff } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-entity-version-diff', + templateUrl: './entity-version-diff.component.html', + styleUrls: ['./entity-version-diff.component.scss'] +}) +export class EntityVersionDiffComponent extends PageComponent implements OnInit, OnDestroy { + + @ViewChild('diffViewer', {static: true}) + diffViewerElmRef: ElementRef; + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: () => void; + + differ: AceDiff; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + getAceDiff().subscribe((aceDiff) => { + this.differ = new aceDiff.default( + { + element: this.diffViewerElmRef.nativeElement, + left: { + copyLinkEnabled: false, + editable: false, + content: 'Left content!' + }, + right: { + copyLinkEnabled: false, + editable: false, + content: 'Right content!' + } + } as AceDiff.AceDiffConstructorOpts + ); + }); + } + + ngOnDestroy(): void { + if (this.differ) { + this.differ.destroy(); + } + } + + close(): void { + if (this.onClose) { + this.onClose(); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 2e5302881e..d931f2f9aa 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -100,10 +100,17 @@ - +
+ + + + {{ 'version-control.differences' | translate : {count: diffCount} }} + + + - - -
-
+
+
{{ 'version-control.current' | translate }}
+
+
{{ versionIdContent() }}
+
+ +
+
- + phone-input.phone-input-label diff --git a/ui-ngx/src/app/shared/components/phone-input.component.ts b/ui-ngx/src/app/shared/components/phone-input.component.ts index 6618811e19..536b1cc723 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.ts +++ b/ui-ngx/src/app/shared/components/phone-input.component.ts @@ -59,6 +59,15 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida @Input() enableFlagsSelect = true; @Input() required = true; + private floatLabel = 'always'; + get showLabel(): string { + return this.floatLabel; + } + @Input() + set showLabel(value) { + this.floatLabel = value ? 'always' : 'never'; + } + allCountries: Array = []; phonePlaceholder: string; countryCallingCode: string; @@ -82,11 +91,9 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida } this.phoneFormGroup = this.fb.group({ country: [this.defaultCountry, []], - phoneNumber: [ - this.countryCallingCode, - this.required ? [Validators.required, Validators.pattern(phoneNumberPattern), this.validatePhoneNumber()] : [] - ] + phoneNumber: [null, this.required ? [Validators.required] : []] }); + this.phoneFormGroup.get('phoneNumber').setValidators([Validators.pattern(phoneNumberPattern), this.validatePhoneNumber()]); this.valueChange$ = this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(() => { this.updateModel(); @@ -97,12 +104,14 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida const code = this.countryCallingCode; this.getFlagAndPhoneNumberData(value); let phoneNumber = this.phoneFormGroup.get('phoneNumber').value; - if (code !== this.countryCallingCode && phoneNumber) { - phoneNumber = phoneNumber.replace(code, this.countryCallingCode); - } else { - phoneNumber = this.countryCallingCode; + if (phoneNumber) { + if (code !== this.countryCallingCode && phoneNumber.includes(code)) { + phoneNumber = phoneNumber.replace(code, this.countryCallingCode); + } else { + phoneNumber = this.countryCallingCode; + } + this.phoneFormGroup.get('phoneNumber').patchValue(phoneNumber); } - this.phoneFormGroup.get('phoneNumber').patchValue(phoneNumber); } }); @@ -123,6 +132,13 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida } } + focus() { + const phoneNumber = this.phoneFormGroup.get('phoneNumber'); + if (!phoneNumber.value) { + phoneNumber.patchValue(this.countryCallingCode); + } + } + getFlagAndPhoneNumberData(country) { if (this.enableFlagsSelect) { this.flagIcon = this.getFlagIcon(country); diff --git a/ui-ngx/src/app/shared/models/country.models.ts b/ui-ngx/src/app/shared/models/country.models.ts index a4ed4d38d9..10612de015 100644 --- a/ui-ngx/src/app/shared/models/country.models.ts +++ b/ui-ngx/src/app/shared/models/country.models.ts @@ -1,3 +1,19 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + import { Injectable } from '@angular/core'; export interface Country { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 050080d5ba..b720594fe0 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3355,7 +3355,7 @@ "phone-input-label": "Phone number", "phone-input-required": "Phone number is required", "phone-input-validation": "Phone number is invalid or not possible", - "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. +19995550123." + "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. {{phoneNumber}}" }, "custom": { "widget-action": { From 34ac17ff83aacce8d0c665605f4b78c3e60f84e9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 30 May 2022 17:32:13 +0300 Subject: [PATCH 130/262] Auto-commit draft --- .../controller/DashboardController.java | 3 +-- .../entitiy/AbstractTbEntityService.java | 3 +++ .../dashboard/DefaultTbDashboardService.java | 3 ++- .../entitiy/dashboard/TbDashboardService.java | 4 +-- .../DefaultEntitiesVersionControlService.java | 26 +++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 2 ++ .../vc/EntitiesVersionControlSettings.java | 10 ++++++- .../create/AutoVersionCreateConfig.java | 25 ++++++++++++++++++ .../request/create/VersionCreateConfig.java | 6 ++++- 9 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 044c61001d..dc4f036852 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -39,7 +39,6 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; -import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HomeDashboard; import org.thingsboard.server.common.data.HomeDashboardInfo; import org.thingsboard.server.common.data.Tenant; @@ -632,7 +631,7 @@ public class DashboardController extends BaseController { DashboardId dashboardId = new DashboardId(toUUID(strDashboardId)); checkDashboardId(dashboardId, Operation.READ); - return tbDashboardService.asignDashboardToEdge(dashboardId, edge, getCurrentUser()); + return tbDashboardService.assignDashboardToEdge(dashboardId, edge, getCurrentUser()); } @ApiOperation(value = "Unassign dashboard from edge (unassignDashboardFromEdge)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index f82203ab3a..7d29500c10 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import javax.mail.MessagingException; import java.util.ArrayList; @@ -108,6 +109,8 @@ public abstract class AbstractTbEntityService { protected QueueService queueService; @Autowired protected DashboardService dashboardService; + @Autowired + protected EntitiesVersionControlService vcService; protected ListenableFuture removeAlarmsByEntityId(TenantId tenantId, EntityId entityId) { ListenableFuture> alarmsFuture = diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index a1bf4baf6b..004d26a314 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -48,6 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement TenantId tenantId = dashboard.getTenantId(); try { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); + vcService.autoCommit(user, savedDashboard.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard, null, actionType, user); return savedDashboard; @@ -219,7 +220,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement } @Override - public Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException { + public Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException { ActionType actionType = ActionType.ASSIGNED_TO_EDGE; TenantId tenantId = user.getTenantId(); EdgeId edgeId = edge.getId(); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java index 84dba247ff..313fd737f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java @@ -26,7 +26,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Set; -public interface TbDashboardService extends SimpleTbEntityService { +public interface TbDashboardService extends SimpleTbEntityService { Dashboard assignDashboardToCustomer(DashboardId dashboardId, Customer customer, SecurityUser user) throws ThingsboardException; @@ -40,7 +40,7 @@ public interface TbDashboardService extends SimpleTbEntityService { Dashboard removeDashboardCustomers(Dashboard dashboard, Set customerIds, SecurityUser user) throws ThingsboardException; - Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException; + Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException; Dashboard unassignDashboardFromEdge(Dashboard dashboard, Edge edge, SecurityUser user) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 0aff14cf7f..864a7365f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -49,6 +50,7 @@ import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; @@ -70,6 +72,9 @@ import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -342,6 +347,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont var future = gitServiceQueue.initRepository(tenantId, restoredSettings); return Futures.transform(future, f -> vcSettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); } catch (Exception e) { + log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); throw new RuntimeException("Failed to init repository!", e); } } @@ -366,6 +372,26 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } + @Override + public ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { + var settings = vcSettingsService.get(user.getTenantId()); + var entityType = entityId.getEntityType(); + if (settings != null && settings.getAutoCommitSettings() != null && settings.getAutoCommitSettings().containsKey(entityType)) { + AutoVersionCreateConfig autoCommitConfig = settings.getAutoCommitSettings().get(entityType); + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(settings.getDefaultBranch()) ? settings.getDefaultBranch() : "auto-commits"; + } + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit by " + user.getEmail() + " at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); + } + return Futures.immediateFuture(null); + } + private String getCauseMessage(Exception e) { String message; if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index a41ea20c68..836a75e021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -61,4 +62,5 @@ public interface EntitiesVersionControlService { ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 3fa361e4ee..7764274d22 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,8 +16,12 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; @Data public class EntitiesVersionControlSettings implements Serializable { @@ -32,7 +36,10 @@ public class EntitiesVersionControlSettings implements Serializable { private String privateKeyPassword; private String defaultBranch; - public EntitiesVersionControlSettings() {} + private Map autoCommitSettings; + + public EntitiesVersionControlSettings() { + } public EntitiesVersionControlSettings(EntitiesVersionControlSettings settings) { this.repositoryUri = settings.getRepositoryUri(); @@ -43,5 +50,6 @@ public class EntitiesVersionControlSettings implements Serializable { this.privateKey = settings.getPrivateKey(); this.privateKeyPassword = settings.getPrivateKeyPassword(); this.defaultBranch = settings.getDefaultBranch(); + this.autoCommitSettings = settings.getAutoCommitSettings() != null ? new HashMap<>(settings.getAutoCommitSettings()) : new HashMap<>(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java new file mode 100644 index 0000000000..f5eb5e1674 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; + +@Data +public class AutoVersionCreateConfig extends VersionCreateConfig { + + private String branch; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index a1aa6cd6d8..10971f1170 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -17,7 +17,11 @@ package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; +import java.io.Serializable; + @Data -public class VersionCreateConfig { +public class VersionCreateConfig implements Serializable { + private static final long serialVersionUID = 1223723167716612772L; + private boolean saveRelations; } From 518d9fd6f5cc93876008aaaa6c056b04749115a6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 30 May 2022 17:34:16 +0300 Subject: [PATCH 131/262] SID for Auto Version Create Config --- .../data/sync/vc/request/create/AutoVersionCreateConfig.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java index f5eb5e1674..084b2a1059 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java @@ -16,10 +16,14 @@ package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class AutoVersionCreateConfig extends VersionCreateConfig { + private static final long serialVersionUID = 8245450889383315551L; + private String branch; } From b6f964a5eb6cd2b9ff40a373a5faa20dca39d377 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 30 May 2022 19:05:17 +0300 Subject: [PATCH 132/262] UI: Implement complex version create action. --- .../home/components/home-components.module.ts | 16 +- .../vc/complex-version-create.component.html | 82 ++++++ ... => complex-version-create.component.scss} | 2 +- .../vc/complex-version-create.component.ts | 104 ++++++++ ...entity-types-version-create.component.html | 111 ++++++++ ...entity-types-version-create.component.scss | 64 +++++ .../entity-types-version-create.component.ts | 247 ++++++++++++++++++ ...l => entity-version-create.component.html} | 8 +- .../vc/entity-version-create.component.scss | 21 ++ ....ts => entity-version-create.component.ts} | 18 +- .../vc/entity-versions-table.component.html | 11 +- .../vc/entity-versions-table.component.ts | 50 +++- .../entity/entity-type-select.component.ts | 26 +- ui-ngx/src/app/shared/models/vc.models.ts | 47 ++++ .../assets/locale/locale.constant-en_US.json | 14 +- 15 files changed, 787 insertions(+), 34 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html rename ui-ngx/src/app/modules/home/components/vc/{entity-version-export.component.scss => complex-version-create.component.scss} (96%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts rename ui-ngx/src/app/modules/home/components/vc/{entity-version-export.component.html => entity-version-create.component.html} (88%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.scss rename ui-ngx/src/app/modules/home/components/vc/{entity-version-export.component.ts => entity-version-create.component.ts} (83%) diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 9afe26d8f1..f0dabc0309 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -156,9 +156,11 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; -import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; +import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; @NgModule({ declarations: @@ -286,9 +288,11 @@ import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-d VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent, + EntityVersionCreateComponent, EntityVersionRestoreComponent, - EntityVersionDiffComponent + EntityVersionDiffComponent, + ComplexVersionCreateComponent, + EntityTypesVersionCreateComponent ], imports: [ CommonModule, @@ -410,9 +414,11 @@ import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-d VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent, + EntityVersionCreateComponent, EntityVersionRestoreComponent, - EntityVersionDiffComponent + EntityVersionDiffComponent, + ComplexVersionCreateComponent, + EntityTypesVersionCreateComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html new file mode 100644 index 0000000000..2c2f5bf274 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -0,0 +1,82 @@ + +
+
+ +

{{ 'version-control.create-entities-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + version-control.default-sync-strategy + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + +
+
+
+
+ + +
+
+
+
{{ resultMessage }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss rename to ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss index 46e55cf4d4..e8b3a5fbf6 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .export-result-message { + .create-result-message { padding: 48px 8px 8px; text-align: center; } diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts new file mode 100644 index 0000000000..c152e6b447 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + ComplexVersionCreateRequest, + createDefaultEntityTypesVersionCreate, + SyncStrategy, syncStrategyTranslationMap, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-complex-version-create', + templateUrl: './complex-version-create.component.html', + styleUrls: ['./complex-version-create.component.scss'] +}) +export class ComplexVersionCreateComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + onContentUpdated: () => void; + + createVersionFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + resultMessage: string; + + versionCreateResult: VersionCreationResult = null; + + versionCreateBranch: string = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + syncStrategy: [SyncStrategy.MERGE, Validators.required], + entityTypes: [createDefaultEntityTypesVersionCreate(), []], + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionCreateResult, this.versionCreateBranch); + } + } + + export(): void { + const request: ComplexVersionCreateRequest = { + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + syncStrategy: this.createVersionFormGroup.get('syncStrategy').value, + entityTypes: this.createVersionFormGroup.get('entityTypes').value, + type: VersionCreateRequestType.COMPLEX + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified && !result.removed) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + } else { + this.resultMessage = this.translate.instant('version-control.version-create-result', + {added: result.added, modified: result.modified, removed: result.removed}); + } + this.versionCreateResult = result; + this.versionCreateBranch = request.branch; + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html new file mode 100644 index 0000000000..74cfe9b891 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -0,0 +1,111 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss new file mode 100644 index 0000000000..f90ad2a803 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-create { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts new file mode 100644 index 0000000000..bf2182dfcd --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -0,0 +1,247 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + SyncStrategy, + syncStrategyTranslationMap +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-entity-types-version-create', + templateUrl: './entity-types-version-create.component.html', + styleUrls: ['./entity-types-version-create.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + } + ] +}) +export class EntityTypesVersionCreateComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionCreateConfig}; + + private propagateChange = null; + + public entityTypesVersionCreateFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionCreateFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), [Validators.min(1)]] + }); + this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionCreateFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionCreateFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionCreateFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionCreateFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + syncStrategy: [config.syncStrategy === null ? 'default' : config.syncStrategy, []], + saveRelations: [config.saveRelations, []], + allEntities: [config.allEntities, []], + entityIds: [config.entityIds, [Validators.required]] + }) + } + ); + this.updateEntityTypeValidators(entityTypeControl); + entityTypeControl.get('config').get('allEntities').valueChanges.subscribe(() => { + this.updateEntityTypeValidators(entityTypeControl); + }); + return entityTypeControl; + } + + private updateEntityTypeValidators(entityTypeControl: AbstractControl): void { + const allEntities: boolean = entityTypeControl.get('config').get('allEntities').value; + if (allEntities) { + entityTypeControl.get('config').get('entityIds').disable({emitEvent: false}); + } else { + entityTypeControl.get('config').get('entityIds').enable({emitEvent: false}); + } + entityTypeControl.get('config').get('entityIds').updateValueAndValidity({emitEvent: false}); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionCreateConfig = { + syncStrategy: null, + saveRelations: false, + allEntities: true, + entityIds: [] + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: EntityTypeVersionCreateConfig = entityTypeControl.get('config').value; + let count = config?.entityIds?.length; + if (!isDefinedAndNotNull(count)) { + count = 0; + } + if (entityType) { + return this.translate.instant((config?.allEntities ? entityTypeTranslations.get(entityType).typePlural + : entityTypeTranslations.get(entityType).list), { count }); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionCreateConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + if ((modelValue[val.entityType].syncStrategy as any) === 'default') { + modelValue[val.entityType].syncStrategy = null; + } + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html similarity index 88% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html rename to ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 3838ea111d..5c575b93d4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -24,7 +24,7 @@ -
+
version-control.version-name - + {{ 'version-control.version-name-required' | translate }} @@ -54,13 +54,13 @@
-
{{ resultMessage }}
+
{{ resultMessage }}
+
+ formGroupName="config" style="min-height: 76px;"> {{ 'version-control.all-entities' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index ca0a8cbe85..8276e12d63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -186,7 +186,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const createVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionCreateComponent, 'left', true, null, + this.viewContainerRef, EntityVersionCreateComponent, 'leftTop', true, null, { branch: this.branch, entityId: this.entityId, @@ -251,7 +251,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const diffVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionDiffComponent, 'left', true, null, + this.viewContainerRef, EntityVersionDiffComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, @@ -275,7 +275,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, From 89c04a3460f1417d66594b714deb6f2ca1396c10 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 11:33:49 +0300 Subject: [PATCH 134/262] UI: Implement complex version load. --- .../home/components/home-components.module.ts | 10 +- .../vc/complex-version-load.component.html | 65 ++++++ .../vc/complex-version-load.component.scss | 27 +++ .../vc/complex-version-load.component.ts | 108 +++++++++ .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 90 ++++++++ .../entity-types-version-load.component.scss | 64 ++++++ .../vc/entity-types-version-load.component.ts | 210 ++++++++++++++++++ .../vc/entity-versions-table.component.html | 10 +- .../vc/entity-versions-table.component.ts | 22 ++ ui-ngx/src/app/shared/models/vc.models.ts | 37 ++- .../assets/locale/locale.constant-en_US.json | 10 +- 12 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f0dabc0309..e9fc3f8c9a 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -161,6 +161,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; +import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @NgModule({ declarations: @@ -292,7 +294,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], imports: [ CommonModule, @@ -418,7 +422,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html new file mode 100644 index 0000000000..200dac7f04 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -0,0 +1,65 @@ + +
+
+ +

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

+ + + + + +
+
+ + +
+
+ +
+ + +
+
+
+
+ {{ 'version-control.no-entities-restored' | translate }} +
+
{{ versionLoadResultMessage(versionLoadResult) }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss new file mode 100644 index 0000000000..eaeb773b2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .load-result-message { + padding: 0 8px; + text-align: center; + &:first-child { + padding-top: 48px; + } + &:nth-last-child(2) { + padding-bottom: 8px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts new file mode 100644 index 0000000000..4b785fd384 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -0,0 +1,108 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + createDefaultEntityTypesVersionLoad, + EntityTypeVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-complex-version-load', + templateUrl: './complex-version-load.component.html', + styleUrls: ['./complex-version-load.component.scss'] +}) +export class ComplexVersionLoadComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + onClose: (result: Array | null) => void; + + @Input() + onContentUpdated: () => void; + + loadVersionFormGroup: FormGroup; + + versionLoadResults: Array = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.loadVersionFormGroup = this.fb.group({ + entityTypes: [createDefaultEntityTypesVersionLoad(), []], + }); + } + + versionLoadResultMessage(result: VersionLoadResult): string { + const entityType = result.entityType; + let message = this.translate.instant(entityTypeTranslations.get(entityType).typePlural) + ': '; + const resultMessages: string[] = []; + if (result.created) { + resultMessages.push(this.translate.instant('version-control.created', {created: result.created})); + } + if (result.updated) { + resultMessages.push(this.translate.instant('version-control.updated', {updated: result.updated})); + } + if (result.deleted) { + resultMessages.push(this.translate.instant('version-control.deleted', {deleted: result.deleted})); + } + message += resultMessages.join(', ') + '.'; + return message; + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionLoadResults); + } + } + + restore(): void { + const request: EntityTypeVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + type: VersionLoadRequestType.ENTITY_TYPE + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + this.versionLoadResults = (result || []).filter(res => res.created || res.updated || res.deleted); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index bf2182dfcd..12fa811dc1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -80,7 +80,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements ngOnInit(): void { this.entityTypesVersionCreateFormGroup = this.fb.group({ - entityTypes: [this.fb.array([]), [Validators.min(1)]] + entityTypes: [this.fb.array([]), []] }); this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { this.updateModel(); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html new file mode 100644 index 0000000000..77cb887d8a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -0,0 +1,90 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.load-entities-relations' | translate }} + +
+
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss new file mode 100644 index 0000000000..b3e840198c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-load { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts new file mode 100644 index 0000000000..39709bebe1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-entity-types-version-load', + templateUrl: './entity-types-version-load.component.html', + styleUrls: ['./entity-types-version-load.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + } + ] +}) +export class EntityTypesVersionLoadComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionLoadConfig}; + + private propagateChange = null; + + public entityTypesVersionLoadFormGroup: FormGroup; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionLoadFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), []] + }); + this.entityTypesVersionLoadFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionLoadFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionLoadFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionLoadFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionLoadFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionLoadConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + loadRelations: [config.loadRelations, []], + removeOtherEntities: [config.removeOtherEntities, []] + }) + } + ); + return entityTypeControl; + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionLoadConfig = { + loadRelations: false, + removeOtherEntities: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + if (entityType) { + return this.translate.instant(entityTypeTranslations.get(entityType).typePlural); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionLoadConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 54f3e69302..b4541c7182 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -107,7 +107,8 @@ - +
@@ -125,6 +126,13 @@ (click)="toggleRestoreEntityVersion($event, restoreVersionButton, entityVersion)"> restore +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 8276e12d63..be714e6f97 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -48,6 +48,7 @@ import { MatButton } from '@angular/material/button'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @Component({ selector: 'tb-entity-versions-table', @@ -291,6 +292,27 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + toggleRestoreEntitiesVersion($event: Event, restoreEntitiesVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreEntitiesVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreEntitiesVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionLoadComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + onClose: (result: Array | null) => { + restoreEntitiesVersionPopover.hide(); + } + }, {}, {}, {}, false); + } + } + versionIdContent(entityVersion: EntityVersion): string { let versionId = entityVersion.id; if (versionId.length > 7) { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 9c5040f5f0..6e66daa24d 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -34,20 +34,11 @@ export interface VersionCreateConfig { saveRelations: boolean; } -export interface VersionLoadConfig { - loadRelations: boolean; -} - export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } -export enum VersionLoadRequestType { - SINGLE_ENTITY = 'SINGLE_ENTITY', - ENTITY_TYPE = 'ENTITY_TYPE' -} - export interface VersionCreateRequest { versionName: string; branch: string; @@ -97,6 +88,15 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: return res; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionLoadRequest { branch: string; versionId: string; @@ -109,6 +109,25 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { type: VersionLoadRequestType.SINGLE_ENTITY; } +export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { + removeOtherEntities: boolean; +} + +export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { + entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; + type: VersionLoadRequestType.ENTITY_TYPE; +} + +export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} { + const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + loadRelations: false, + removeOtherEntities: false + }; + } + return res; +} export interface BranchInfo { name: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5dfece08ed..49b4f82c17 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3144,12 +3144,18 @@ "sync-strategy-overwrite": "Overwrite", "entity-types": "Entity types", "sync-strategy": "Sync strategy", - "default": "Default", "all-entities": "All entities", "no-entity-types": "No entity types configured", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed." + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "load-entities-relations": "Load entities relations", + "remove-other-entities": "Remove other entities", + "restore-entities-from-version": "Restore entities from version '{{versionName}}'", + "no-entities-restored": "No entities restored", + "created": "{{created}} created", + "updated": "{{updated}} updated", + "deleted": "{{deleted}} deleted" }, "widget": { "widget-library": "Widgets Library", From bb9b58da5b13306bf32a2c0884e0d006f1b6e839 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 13:09:47 +0300 Subject: [PATCH 135/262] Add author to entity version --- .../DefaultEntitiesVersionControlService.java | 2 +- .../DefaultGitVersionControlQueueService.java | 35 +++++++++++++------ .../vc/GitVersionControlQueueService.java | 3 +- common/cluster-api/src/main/proto/queue.proto | 10 ++++-- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 6 ++-- .../sync/vc/DefaultGitRepositoryService.java | 12 +++++-- .../server/service/sync/vc/GitRepository.java | 7 ++-- .../server/service/sync/vc/PendingCommit.java | 8 ++++- .../vc/entity-versions-table.component.html | 8 ++++- .../vc/entity-versions-table.component.ts | 2 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 1 + 13 files changed, 71 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..0dc90fd109 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -120,7 +120,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings("UnstableApiUsage") @Override public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); + var pendingCommit = gitServiceQueue.prepareCommit(user, request); return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..891c68b802 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -66,10 +67,7 @@ import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -96,12 +94,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + public ListenableFuture prepareCommit(User user, VersionCreateRequest request) { SettableFuture future = SettableFuture.create(); - CommitGitRequest commit = new CommitGitRequest(tenantId, request); + CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); registerAndSend(commit, builder -> builder.setCommitRequest( - buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(request)).build() + buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() ).build(), wrap(future, commit)); return future; } @@ -356,7 +354,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -405,7 +403,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { @@ -453,8 +451,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return path; } - private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { - return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()) + .setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build(); + } + + private static String getAuthorName(User user) { + List parts = new ArrayList<>(); + if (StringUtils.isNotBlank(user.getFirstName())) { + parts.add(user.getFirstName()); + } + if (StringUtils.isNotBlank(user.getLastName())) { + parts.add(user.getLastName()); + } + if (parts.isEmpty()) { + parts.add(user.getName()); + } + return String.join(" ", parts); } private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..5a4ffe1706 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -36,7 +37,7 @@ import java.util.List; public interface GitVersionControlQueueService { - ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + ListenableFuture prepareCommit(User user, VersionCreateRequest request); ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 969ee0261c..3079978bce 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -692,14 +692,17 @@ message CommitResponseMsg { int64 ts = 1; string commitId = 2; string name = 3; - int32 added = 4; - int32 modified = 5; - int32 removed = 6; + string author = 4; + int32 added = 5; + int32 modified = 6; + int32 removed = 7; } message PrepareMsg { string commitMsg = 1; string branchName = 2; + string authorName = 3; + string authorEmail = 4; } message AddMsg { @@ -733,6 +736,7 @@ message EntityVersionProto { int64 ts = 1; string id = 2; string name = 3; + string author = 4; } message ListVersionsResponseMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 50d3fef8a8..c90336d3d7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -26,4 +26,5 @@ public class EntityVersion { private long timestamp; private String id; private String name; + private String author; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..dc42a61257 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -312,7 +312,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).setAuthor(v.getAuthor()).build() ).collect(Collectors.toList()))) ); } @@ -397,7 +397,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void prepareCommit(VersionControlRequestCtx ctx, UUID txId, PrepareMsg prepareMsg) { var tenantId = ctx.getTenantId(); - var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), prepareMsg.getCommitMsg()); + var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), + prepareMsg.getCommitMsg(), prepareMsg.getAuthorName(), prepareMsg.getAuthorEmail()); PendingCommit old = pendingCommitMap.get(tenantId); if (old != null) { doAbortCurrentCommit(tenantId, old); @@ -460,6 +461,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()) .setAdded(result.getAdded()) .setModified(result.getModified()) .setRemoved(result.getRemoved()))); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..8c30d3097a 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,7 +117,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName()); + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); repository.push(commit.getWorkingBranch(), commit.getBranch()); result.setVersion(toVersion(gitCommit)); @@ -255,7 +255,15 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage(), this.getAuthor(commit)); + } + + private String getAuthor(GitRepository.Commit commit) { + String author = String.format("<%s>", commit.getAuthorEmail()); + if (StringUtils.isNotBlank(commit.getAuthorName())) { + author = String.format("%s %s", commit.getAuthorName(), author); + } + return author; } public static EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..938094cfb8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -255,8 +255,9 @@ public class GitRepository { return new Status(status.getAdded(), modified, status.getRemoved()); } - public Commit commit(String message) throws GitAPIException { + public Commit commit(String message, String authorName, String authorEmail) throws GitAPIException { RevCommit revCommit = execute(git.commit() + .setAuthor(authorName, authorEmail) .setMessage(message)); return toCommit(revCommit); } @@ -325,7 +326,8 @@ public class GitRepository { } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), + revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); } private RevCommit resolveCommit(String id) throws IOException { @@ -470,6 +472,7 @@ public class GitRepository { private final String id; private final String message; private final String authorName; + private final String authorEmail; } @Data diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index b2eccd79db..ccd5fc685e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -30,12 +30,18 @@ public class PendingCommit { private String branch; private String versionName; - public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName) { + private String authorName; + + private String authorEmail; + + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { this.tenantId = tenantId; this.nodeId = nodeId; this.txId = txId; this.branch = branch; this.versionName = versionName; + this.authorName = authorName; + this.authorEmail = authorEmail; this.workingBranch = txId.toString(); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b4541c7182..d2dbc119bf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -101,11 +101,17 @@ - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} + + {{ 'version-control.author' | translate }} + + {{ entityVersion.author }} + + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index be714e6f97..589800e143 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -62,7 +62,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name', 'actions']; + displayedColumns = ['timestamp', 'id', 'name', 'author', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6e66daa24d..0ad52d2e0b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -138,6 +138,7 @@ export interface EntityVersion { timestamp: number; id: string; name: string; + author: string; } export interface VersionCreationResult { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 49b4f82c17..4a4e228cc2 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3119,6 +3119,7 @@ "create-entity-version": "Create entity version", "version-name": "Version name", "version-name-required": "Version name is required", + "author": "Author", "export-entity-relations": "Export entity relations", "entity-versions": "Entity versions", "versions": "Versions", From 07b6813465f958db07a9945f347a2f59b45af1d5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 13:19:16 +0300 Subject: [PATCH 136/262] Separate AutoCommit settings --- .../server/controller/AdminController.java | 72 +++++++++++-- .../DefaultEntitiesVersionControlService.java | 62 ++++++----- .../DefaultGitVersionControlQueueService.java | 12 +-- ...efaultTbVersionControlSettingsService.java | 102 ------------------ .../vc/EntitiesVersionControlService.java | 9 +- .../vc/GitVersionControlQueueService.java | 6 +- ...AbstractVersionControlSettingsService.java | 80 ++++++++++++++ .../AutoCommitSettingsCaffeineCache.java} | 14 ++- .../AutoCommitSettingsRedisCache.java} | 17 ++- .../DefaultTbAutoCommitSettingsService.java | 36 +++++++ .../TbAutoCommitSettingsService.java | 30 ++++++ .../DefaultTbRepositorySettingsService.java | 63 +++++++++++ .../RepositorySettingsCaffeineCache.java | 34 ++++++ .../RepositorySettingsRedisCache.java | 36 +++++++ .../TbRepositorySettingsService.java} | 12 +-- .../src/main/resources/thingsboard.yml | 9 +- .../server/common/data/CacheConstants.java | 3 +- .../data/sync/vc/AutoCommitSettings.java | 27 +++++ ...lSettings.java => RepositorySettings.java} | 11 +- .../DefaultClusterVersionControlService.java | 6 +- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../server/service/sync/vc/GitRepository.java | 12 +-- .../service/sync/vc/GitRepositoryService.java | 8 +- .../sync/vc/VersionControlRequestCtx.java | 6 +- 24 files changed, 467 insertions(+), 208 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsCaffeineCache.java => autocommit/AutoCommitSettingsCaffeineCache.java} (66%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsRedisCache.java => autocommit/AutoCommitSettingsRedisCache.java} (58%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{TbVersionControlSettingsService.java => repository/TbRepositorySettingsService.java} (60%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{EntitiesVersionControlSettings.java => RepositorySettings.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index cbc6ff2bca..67665ad8c7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -29,24 +29,23 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.*; -import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -68,6 +67,9 @@ public class AdminController extends BaseController { @Autowired private EntitiesVersionControlService versionControlService; + @Autowired + private TbAutoCommitSettingsService autoCommitSettingsService; + @Autowired private UpdateService updateService; @@ -194,10 +196,10 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vcSettings") @ResponseBody - public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getVersionControlSettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); versionControlSettings.setPassword(null); versionControlSettings.setPrivateKey(null); versionControlSettings.setPrivateKeyPassword(null); @@ -225,9 +227,9 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); + ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { savedSettings.setPassword(null); savedSettings.setPrivateKey(null); @@ -251,13 +253,65 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", + notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings") + @ResponseBody + public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return checkNotNull(autoCommitSettingsService.get(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings/exists") + @ResponseBody + public Boolean autoCommitSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return autoCommitSettingsService.get(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vc/autoCommitSettings") + public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + return autoCommitSettingsService.save(getTenantId(), settings); + } + + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + autoCommitSettingsService.delete(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) public DeferredResult checkVersionControlAccess( @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + @RequestBody RepositorySettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); settings = checkNotNull(settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..3060b05b1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -27,7 +27,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -44,7 +43,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -62,19 +61,18 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -94,7 +92,8 @@ import static com.google.common.util.concurrent.Futures.transformAsync; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final TbVersionControlSettingsService vcSettingsService; + private final TbRepositorySettingsService repositorySettingsService; + private final TbAutoCommitSettingsService autoCommitSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; @@ -336,16 +335,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - return vcSettingsService.get(tenantId); + public RepositorySettings getVersionControlSettings(TenantId tenantId) { + return repositorySettingsService.get(tenantId); } @Override - public ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - var restoredSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); + public ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) { + var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings); try { var future = gitServiceQueue.initRepository(tenantId, restoredSettings); - return Futures.transform(future, f -> vcSettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); + return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); } catch (Exception e) { log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); throw new RuntimeException("Failed to init repository!", e); @@ -354,7 +353,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (vcSettingsService.delete(tenantId)) { + if (repositorySettingsService.delete(tenantId)) { return gitServiceQueue.clearRepository(tenantId); } else { return Futures.immediateFuture(null); @@ -362,8 +361,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - settings = this.vcSettingsService.restore(tenantId, settings); + public ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException { + settings = this.repositorySettingsService.restore(tenantId, settings); try { return gitServiceQueue.testRepository(tenantId, settings); } catch (Exception e) { @@ -374,22 +373,29 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { - var settings = vcSettingsService.get(user.getTenantId()); + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } var entityType = entityId.getEntityType(); - if (settings != null && settings.getAutoCommitSettings() != null && settings.getAutoCommitSettings().containsKey(entityType)) { - AutoVersionCreateConfig autoCommitConfig = settings.getAutoCommitSettings().get(entityType); - SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); - var autoCommitBranchName = autoCommitConfig.getBranch(); - if (StringUtils.isEmpty(autoCommitBranchName)) { - autoCommitBranchName = StringUtils.isNotEmpty(settings.getDefaultBranch()) ? settings.getDefaultBranch() : "auto-commits"; - } - vcr.setBranch(autoCommitBranchName); - vcr.setVersionName("auto-commit by " + user.getEmail() + " at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); - vcr.setEntityId(entityId); - vcr.setConfig(autoCommitConfig); - return saveEntitiesVersion(user, vcr); + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; } - return Futures.immediateFuture(null); + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); } private String getCauseMessage(Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..7f501522af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -271,7 +271,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private void registerAndSend(PendingGitRequest request, - Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { + Function enrichFunction, RepositorySettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); var requestBody = enrichFunction.apply(newRequestProto(request, settings)); @@ -307,7 +307,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() @@ -317,7 +317,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder @@ -457,7 +457,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); } - private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, RepositorySettings settings) { var tenantId = request.getTenantId(); var requestId = request.getRequestId(); var builder = ToVersionControlServiceMsg.newBuilder() @@ -466,7 +466,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()); - EntitiesVersionControlSettings vcSettings = settings; + RepositorySettings vcSettings = settings; if (vcSettings == null && request.requiresSettings()) { vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java deleted file mode 100644 index 1d1fff81a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { - - public static final String SETTINGS_KEY = "entitiesVersionControl"; - private final AdminSettingsService adminSettingsService; - private final TbTransactionalCache cache; - - @Override - public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { - EntitiesVersionControlSettings storedSettings = get(tenantId); - if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } - - @Override - public EntitiesVersionControlSettings get(TenantId tenantId) { - EntitiesVersionControlSettings settings = cache.getAndPutInTransaction(tenantId, () -> { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - }, true); - if (settings != null) { - settings = new EntitiesVersionControlSettings(settings); - } - return settings; - } - - @Override - public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. - cache.evict(tenantId); - return savedVersionControlSettings; - } - - @Override - public boolean delete(TenantId tenantId) { - boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); - cache.evict(tenantId); - return result; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 836a75e021..447ae1d021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -24,7 +23,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -54,13 +53,13 @@ public interface EntitiesVersionControlService { ListenableFuture> listBranches(TenantId tenantId) throws Exception; - EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + RepositorySettings getVersionControlSettings(TenantId tenantId); - ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings); ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception; - ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..221206f5bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -64,9 +64,9 @@ public interface GitVersionControlQueueService { ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); - ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); - ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); ListenableFuture clearRepository(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java new file mode 100644 index 0000000000..e826c32c56 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.io.Serializable; + +public abstract class TbAbstractVersionControlSettingsService { + + private final String settingsKey; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + private final Class clazz; + + public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache, Class clazz, String settingsKey) { + this.adminSettingsService = adminSettingsService; + this.cache = cache; + this.clazz = clazz; + this.settingsKey = settingsKey; + } + + public T get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load " + settingsKey + " settings!", e); + } + } + return null; + }, true); + } + + public T save(TenantId tenantId, T settings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(settingsKey); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + T savedSettings; + try { + savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load auto commit settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedSettings; + } + + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, settingsKey); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java index 8cab03ca29..47b9e5f5d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java @@ -13,24 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CaffeineTbTransactionalCache; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) -@Service("VersionControlCache") -public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache { - public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { - super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java index 9c778d6e4f..f88b1cf6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -13,29 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; -import com.google.protobuf.InvalidProtocolBufferException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.RedisSerializer; -import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") -@Service("VersionControlCache") -public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { - public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java new file mode 100644 index 0000000000..b6e8d45ec2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService implements TbAutoCommitSettingsService { + + public static final String SETTINGS_KEY = "autoCommitSettings"; + + public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java new file mode 100644 index 0000000000..51978481e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbAutoCommitSettingsService { + + AutoCommitSettings get(TenantId tenantId); + + AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java new file mode 100644 index 0000000000..608efec870 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbRepositorySettingsService extends TbAbstractVersionControlSettingsService implements TbRepositorySettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + + public DefaultTbRepositorySettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, RepositorySettings.class, SETTINGS_KEY); + } + + @Override + public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { + RepositorySettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public RepositorySettings get(TenantId tenantId) { + RepositorySettings settings = super.get(tenantId); + if (settings != null) { + settings = new RepositorySettings(settings); + } + return settings; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java new file mode 100644 index 0000000000..60b7f50e12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RepositorySettingsCache") +public class RepositorySettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public RepositorySettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.REPOSITORY_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java new file mode 100644 index 0000000000..3cccb6d24e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RepositorySettingsCache") +public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { + + public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java index 178499c6c7..946d06c87c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.repository; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -public interface TbVersionControlSettingsService { +public interface TbRepositorySettingsService { - EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings restore(TenantId tenantId, RepositorySettings versionControlSettings); - EntitiesVersionControlSettings get(TenantId tenantId); + RepositorySettings get(TenantId tenantId); - EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings save(TenantId tenantId, RepositorySettings versionControlSettings); boolean delete(TenantId tenantId); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 31e5d7a76e..9663396d7e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,9 +433,12 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" - vcSettings: - timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" + repositorySettings: + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + autoCommitSettings: + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 85ea116f0a..94ca9db5a5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,5 +31,6 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; - public static final String VC_SETTINGS_CACHE = "vcSettings"; + public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings"; + public static final String AUTO_COMMIT_SETTINGS_CACHE = "autoCommitSettings"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java new file mode 100644 index 0000000000..2120ec4c44 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; + +import java.util.HashMap; + +public class AutoCommitSettings extends HashMap { + + private static final long serialVersionUID = -5757067601838792059L; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java similarity index 80% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index 7764274d22..c576f82298 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -24,7 +25,8 @@ import java.util.HashMap; import java.util.Map; @Data -public class EntitiesVersionControlSettings implements Serializable { +@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. +public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; @@ -36,12 +38,10 @@ public class EntitiesVersionControlSettings implements Serializable { private String privateKeyPassword; private String defaultBranch; - private Map autoCommitSettings; - - public EntitiesVersionControlSettings() { + public RepositorySettings() { } - public EntitiesVersionControlSettings(EntitiesVersionControlSettings settings) { + public RepositorySettings(RepositorySettings settings) { this.repositoryUri = settings.getRepositoryUri(); this.authMethod = settings.getAuthMethod(); this.username = settings.getUsername(); @@ -50,6 +50,5 @@ public class EntitiesVersionControlSettings implements Serializable { this.privateKey = settings.getPrivateKey(); this.privateKeyPassword = settings.getPrivateKeyPassword(); this.defaultBranch = settings.getDefaultBranch(); - this.autoCommitSettings = settings.getAutoCommitSettings() != null ? new HashMap<>(settings.getAutoCommitSettings()) : new HashMap<>(); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..9463608b76 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -495,8 +495,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } - private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { - Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + private RepositorySettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); if (settingsOpt.isPresent()) { return settingsOpt.get(); } else { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..b12e03e762 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -216,13 +216,13 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository.test(settings, repositoryDirectory.toFile()); } @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception { clearRepository(tenantId); log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); @@ -238,7 +238,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception { + public RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception { var gitRepository = repositories.get(tenantId); return gitRepository != null ? gitRepository.getSettings() : null; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..b5bc6355d4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -58,7 +58,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; @@ -77,14 +77,14 @@ public class GitRepository { private final Git git; @Getter - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; private final CredentialsProvider credentialsProvider; private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, EntitiesVersionControlSettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.settings = settings; this.credentialsProvider = credentialsProvider; @@ -92,7 +92,7 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { @@ -109,7 +109,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { + public static GitRepository open(File directory, RepositorySettings settings) throws IOException { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; @@ -121,7 +121,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index ec69e0c80e..057fa9c1ab 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -19,7 +19,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -39,11 +39,11 @@ public interface GitRepositoryService { List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception; + RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception; void clearRepository(TenantId tenantId) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java index 6541a2978a..26ab0d94c1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.sync.vc; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import java.util.UUID; @@ -29,9 +29,9 @@ public class VersionControlRequestCtx { private final String nodeId; private final UUID requestId; private final TenantId tenantId; - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; - public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, RepositorySettings settings) { this.nodeId = msg.getNodeId(); this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); From 32f2f4ccd07c044b9396c42b556146c4daa3c832 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 13:19:36 +0300 Subject: [PATCH 137/262] Export/import of attributes --- .../impl/DefaultEntityExportService.java | 79 ++++++++++++--- .../impl/BaseEntityImportService.java | 97 +++++++++++++++---- .../DefaultEntitiesVersionControlService.java | 15 ++- .../data/sync/ie/AttributeExportData.java | 30 ++++++ .../common/data/sync/ie/EntityExportData.java | 2 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 9 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 31bf768e90..8e723f8793 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -19,22 +19,31 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; @Service @TbCoreComponent @@ -45,6 +54,8 @@ public class DefaultEntityExportService relations = new ArrayList<>(); + List relations = exportRelations(user, entity); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(user, entity); + exportData.setAttributes(attributes); + } + } - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - relations.addAll(inboundRelations); + private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - relations.addAll(outboundRelations); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + relations.addAll(inboundRelations); - exportData.setRelations(relations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES); + + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new HashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 203b0caa16..caa1e20ad7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -29,23 +32,36 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; +@Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -53,6 +69,8 @@ public abstract class BaseEntityImportService { - if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { - return; - } - - List relations = new ArrayList<>(exportData.getRelations()); + if (importSettings.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(user, exportData.getRelations(), importResult); + } + if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) { + importAttributes(user, exportData.getAttributes(), importResult); + } + } + private void importRelations(SecurityUser user, List relations, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { - if (!relation.getTo().equals(savedEntity.getId())) { + if (!relation.getTo().equals(entity.getId())) { HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { + if (!relation.getFrom().equals(entity.getId())) { HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); } } - if (oldEntity != null) { + if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { EntityId otherEntity = null; - if (!existingRelation.getTo().equals(savedEntity.getId())) { + if (!existingRelation.getTo().equals(entity.getId())) { otherEntity = existingRelation.getTo(); - } else if (!existingRelation.getFrom().equals(savedEntity.getId())) { + } else if (!existingRelation.getFrom().equals(entity.getId())) { otherEntity = existingRelation.getFrom(); } if (otherEntity != null) { @@ -161,6 +183,44 @@ public abstract class BaseEntityImportService> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) {} + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), @@ -168,6 +228,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) @@ -181,6 +242,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) @@ -216,7 +278,8 @@ public abstract class BaseEntityImportService saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -208,6 +208,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(false) .build(), true, true); } catch (Exception e) { @@ -246,6 +247,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(config.isFindExistingEntityByName()) .build(), false, false); @@ -283,6 +285,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); VersionLoadResult result = results.get(entityType); result.setDeleted(result.getDeleted() + 1); } @@ -321,6 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(true) + .exportAttributes(true) .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 35fceafe43..542e675940 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; +import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @@ -45,5 +46,6 @@ public class EntityExportData> { private EntityType entityType; private List relations; + private Map> attributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 051745c07f..0800a1f7c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportRelations; + private boolean exportAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 3e18bc9f5f..564b0134e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 10971f1170..4426717fe9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -24,4 +24,5 @@ public class VersionCreateConfig implements Serializable { private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 8ed9c34694..2d68a2b7e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,5 +21,6 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; + private boolean loadAttributes; } From 09402f40acb56b7f0d14ade095c4c509cabebd36 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 16:16:24 +0300 Subject: [PATCH 138/262] UI: Refactor vc settings to repository settings. Add remove other entities confirm. --- .../server/controller/AdminController.java | 77 ++++--- .../DefaultTbRepositorySettingsService.java | 8 +- ...hMethod.java => RepositoryAuthMethod.java} | 2 +- .../data/sync/vc/RepositorySettings.java | 6 +- .../server/service/sync/vc/GitRepository.java | 14 +- ui-ngx/src/app/core/auth/auth.actions.ts | 10 +- ui-ngx/src/app/core/auth/auth.models.ts | 2 +- ui-ngx/src/app/core/auth/auth.reducer.ts | 4 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 4 +- ui-ngx/src/app/core/auth/auth.service.ts | 10 +- ui-ngx/src/app/core/http/admin.service.ts | 47 ++-- .../http/entities-version-control.service.ts | 8 +- ui-ngx/src/app/core/services/menu.service.ts | 8 +- .../home/components/home-components.module.ts | 13 +- ...entity-types-version-create.component.html | 13 +- .../entity-types-version-create.component.ts | 2 + .../entity-types-version-load.component.html | 28 ++- .../vc/entity-types-version-load.component.ts | 42 +++- .../vc/entity-version-create.component.html | 3 + .../vc/entity-version-create.component.ts | 6 +- .../vc/entity-version-restore.component.html | 3 + .../vc/entity-version-restore.component.ts | 6 +- ...move-other-entities-confirm.component.html | 41 ++++ ...remove-other-entities-confirm.component.ts | 66 ++++++ ...tml => repository-settings.component.html} | 26 +-- ...css => repository-settings.component.scss} | 2 +- .../vc/repository-settings.component.ts | 216 +++++++++++++++++ .../vc/version-control-settings.component.ts | 218 ------------------ .../vc/version-control.component.html | 6 +- .../vc/version-control.component.ts | 10 +- .../home/pages/admin/admin-routing.module.ts | 10 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ... repository-admin-settings.component.html} | 2 +- ...=> repository-admin-settings.component.ts} | 12 +- ui-ngx/src/app/shared/models/constants.ts | 2 +- .../src/app/shared/models/settings.models.ts | 19 +- ui-ngx/src/app/shared/models/vc.models.ts | 23 +- .../assets/locale/locale.constant-en_US.json | 19 +- 38 files changed, 592 insertions(+), 400 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{VersionControlAuthMethod.java => RepositoryAuthMethod.java} (94%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.html => repository-settings.component.html} (78%) rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.scss => repository-settings.component.scss} (96%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts delete mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.html => repository-admin-settings.component.html} (87%) rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.ts => repository-admin-settings.component.ts} (68%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 67665ad8c7..ed740f73fb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -191,12 +191,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Get version control settings (getVersionControlSettings)", - notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Get repository settings (getRepositorySettings)", + notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings") + @GetMapping("/repositorySettings") @ResponseBody - public RepositorySettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); @@ -209,12 +209,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check repository settings exists (repositorySettingsExists)", + notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings/exists") + @GetMapping("/repositorySettings/exists") @ResponseBody - public Boolean versionControlSettingsExists() throws ThingsboardException { + public Boolean repositorySettingsExists() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); return versionControlService.getVersionControlSettings(getTenantId()) != null; @@ -223,11 +223,11 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)", + notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { + @PostMapping("/repositorySettings") + public DeferredResult saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { @@ -238,13 +238,13 @@ public class AdminController extends BaseController { }, MoreExecutors.directExecutor())); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete repository settings (deleteRepositorySettings)", + notes = "Deletes the repository settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) - public DeferredResult deleteVersionControlSettings() throws ThingsboardException { + public DeferredResult deleteRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId())); @@ -253,6 +253,23 @@ public class AdminController extends BaseController { } } + + @ApiOperation(value = "Check repository access (checkRepositoryAccess)", + notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST) + public DeferredResult checkRepositoryAccess( + @ApiParam(value = "A JSON value representing the Repository Settings.") + @RequestBody RepositorySettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -267,8 +284,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", + notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vc/autoCommitSettings/exists") @ResponseBody @@ -281,8 +298,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", + notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vc/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { @@ -290,8 +307,8 @@ public class AdminController extends BaseController { return autoCommitSettingsService.save(getTenantId(), settings); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)", + notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) @@ -305,22 +322,6 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control access (checkVersionControlAccess)", - notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) - public DeferredResult checkVersionControlAccess( - @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody RepositorySettings settings) throws ThingsboardException { - try { - accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - settings = checkNotNull(settings); - return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); - } catch (Exception e) { - throw handleException(e); - } - } - @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java index 608efec870..555490bd06 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -19,7 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; @@ -38,10 +38,10 @@ public class DefaultTbRepositorySettingsService extends TbAbstractVersionControl public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { RepositorySettings storedSettings = get(tenantId); if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + RepositoryAuthMethod authMethod = settings.getAuthMethod(); + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { settings.setPrivateKey(storedSettings.getPrivateKey()); if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java index 770a1e582c..3c0e5849a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; -public enum VersionControlAuthMethod { +public enum RepositoryAuthMethod { USERNAME_PASSWORD, PRIVATE_KEY } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index c576f82298..eb1a97ff6c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -17,12 +17,8 @@ package org.thingsboard.server.common.data.sync.vc; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; @Data @JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. @@ -30,7 +26,7 @@ public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; - private VersionControlAuthMethod authMethod; + private RepositoryAuthMethod authMethod; private String username; private String password; private String privateKeyFileName; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dd9a67aed4..2b2802d003 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -59,7 +59,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -95,9 +95,9 @@ public class GitRepository { public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } CloneCommand cloneCommand = Git.cloneRepository() @@ -113,9 +113,9 @@ public class GitRepository { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); @@ -124,9 +124,9 @@ public class GitRepository { public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 872607b822..a60720e61d 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -24,7 +24,7 @@ export enum AuthActionTypes { LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', - UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' + UPDATE_HAS_REPOSITORY = '[Auth] Change Has Repository' } export class ActionAuthAuthenticated implements Action { @@ -55,11 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } -export class ActionAuthUpdateHasVersionControl implements Action { - readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; +export class ActionAuthUpdateHasRepository implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_REPOSITORY; - constructor(readonly payload: { hasVersionControl: boolean }) {} + constructor(readonly payload: { hasRepository: boolean }) {} } export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasRepository; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index a69b783526..33b84945aa 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,7 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; - hasVersionControl: boolean; + hasRepository: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 9d9404524b..cace62660e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -24,7 +24,7 @@ const emptyUserAuthState: AuthPayload = { forceFullscreen: false, allowedDashboardIds: [], edgesSupportEnabled: false, - hasVersionControl: false + hasRepository: false }; export const initialState: AuthState = { @@ -55,7 +55,7 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; - case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + case AuthActionTypes.UPDATE_HAS_REPOSITORY: return { ...state, ...action.payload}; default: diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index 7a099d5acd..4a63dbcbed 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,9 +55,9 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); -export const selectHasVersionControl = createSelector( +export const selectHasRepository = createSelector( selectAuthState, - (state: AuthState) => state.hasVersionControl + (state: AuthState) => state.hasRepository ); export function getCurrentAuthState(store: Store): AuthState { diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6fe809955f..74f6689553 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,9 +437,9 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } - private loadHasVersionControl(authUser: AuthUser): Observable { + private loadHasRepository(authUser: AuthUser): Observable { if (authUser.authority === Authority.TENANT_ADMIN) { - return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + return this.http.get('/api/admin/repositorySettings/exists', defaultHttpOptions()); } else { return of(false); } @@ -449,15 +449,15 @@ export class AuthService { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), - this.loadHasVersionControl(authPayload.authUser), + this.loadHasRepository(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - const hasVersionControl: boolean = data[3] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; + const hasRepository: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasRepository}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 7c12bd3a5d..38f18bd598 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -15,19 +15,21 @@ /// import { Injectable } from '@angular/core'; -import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, - EntitiesVersionControlSettings, + RepositorySettings, MailServerSettings, SecuritySettings, TestSmsRequest, - UpdateMessage + UpdateMessage, AutoCommitSettings } from '@shared/models/settings.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { tap } from 'rxjs/operators'; +import { AuthUser } from '@shared/models/user.model'; +import { Authority } from '@shared/models/authority.enum'; @Injectable({ providedIn: 'root' @@ -68,13 +70,13 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } - public getEntitiesVersionControlSettings(config?: RequestConfig): Observable { - return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + public getRepositorySettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); } - public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings', versionControlSettings, + public saveRepositorySettings(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings', repositorySettings, defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); @@ -82,17 +84,34 @@ export class AdminService { ); } - public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + public deleteRepositorySettings(config?: RequestConfig) { + return this.http.delete('/api/admin/repositorySettings', defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); }) ); } - public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + public checkRepositoryAccess(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings/checkAccess', repositorySettings, defaultHttpOptionsFromConfig(config)); + } + + public getAutoCommitSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); + } + + private autoCommitSettingsExists(config?: RequestConfig): Observable { + return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); + } + + public saveAutoCommitSettings(autoCommitSettings: AutoCommitSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/autoCommitSettings', autoCommitSettings, defaultHttpOptionsFromConfig(config)); + } + + public deleteAutoCommitSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/autoCommitSettings', defaultHttpOptionsFromConfig(config)); } public checkUpdates(config?: RequestConfig): Observable { diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 7578696774..9f6f4f3403 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,7 +17,7 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { combineLatest, Observable, of } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { BranchInfo, EntityDataDiff, EntityVersion, @@ -29,10 +29,10 @@ import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; -import { createSelector, select, Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; -import { catchError, combineAll, tap } from 'rxjs/operators'; +import { selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index ee81f007ee..50938e5869 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -388,9 +388,9 @@ export class MenuService { }, { id: guid(), - name: 'admin.git-settings', + name: 'admin.repository-settings', type: 'link', - path: '/settings/vc', + path: '/settings/repository', icon: 'manage_history' } ] @@ -538,9 +538,9 @@ export class MenuService { path: '/settings/resources-library' }, { - name: 'admin.git-settings', + name: 'admin.repository-settings', icon: 'manage_history', - path: '/settings/vc', + path: '/settings/repository', } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index e9fc3f8c9a..613f236cf6 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -153,7 +153,7 @@ import { TenantProfileQueuesComponent } from '@home/components/profile/queue/ten import { QueueFormComponent } from '@home/components/queue/queue-form.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; @@ -163,6 +163,7 @@ import { ComplexVersionCreateComponent } from '@home/components/vc/complex-versi import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @NgModule({ declarations: @@ -287,7 +288,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -296,7 +297,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], imports: [ CommonModule, @@ -415,7 +417,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -424,7 +426,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 9d1192c0c9..3c1dd73117 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -53,7 +53,7 @@ [allowedEntityTypes]="allowedEntityTypes(entityTypeFormGroup)">
- + version-control.sync-strategy @@ -64,9 +64,14 @@ - - {{ 'version-control.export-entity-relations' | translate }} - +
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
-
+
-
- - {{ 'version-control.remove-other-entities' | translate }} - - - {{ 'version-control.load-entities-relations' | translate }} - +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} + +
+
+ + {{ 'version-control.load-entities-relations' | translate }} + + + {{ 'version-control.load-entities-attributes' | translate }} + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 39709bebe1..40bf9ad869 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { Component, forwardRef, Input, OnInit, Renderer2, ViewContainerRef } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -28,11 +28,15 @@ import { Validators } from '@angular/forms'; import { PageComponent } from '@shared/components/page.component'; -import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes, VersionCreationResult } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { TranslateService } from '@ngx-translate/core'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { MatCheckbox } from '@angular/material/checkbox/checkbox'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @Component({ selector: 'tb-entity-types-version-load', @@ -64,6 +68,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On constructor(protected store: Store, private translate: TranslateService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private viewContainerRef: ViewContainerRef, private fb: FormBuilder) { super(store); } @@ -124,7 +131,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On entityType: [entityType, [Validators.required]], config: this.fb.group({ loadRelations: [config.loadRelations, []], - removeOtherEntities: [config.removeOtherEntities, []] + loadAttributes: [config.loadAttributes, []], + removeOtherEntities: [config.removeOtherEntities, []], + findExistingEntityByName: [config.findExistingEntityByName, []] }) } ); @@ -156,7 +165,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -194,6 +205,29 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On return res; } + onRemoveOtherEntities(removeOtherEntitiesCheckbox: MatCheckbox, entityTypeControl: AbstractControl, $event: Event) { + const removeOtherEntities: boolean = entityTypeControl.get('config.removeOtherEntities').value; + if (!removeOtherEntities) { + $event.preventDefault(); + $event.stopPropagation(); + const trigger = $('.mat-checkbox-frame', removeOtherEntitiesCheckbox._elementRef.nativeElement)[0]; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const removeOtherEntitiesConfirmPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, RemoveOtherEntitiesConfirmComponent, 'bottom', true, null, + { + onClose: (result: boolean | null) => { + removeOtherEntitiesConfirmPopover.hide(); + if (result) { + entityTypeControl.get('config').get('removeOtherEntities').patchValue(true, {emitEvent: true}); + } + } + }, {}, {}, {}, false); + } + } + } + private updateModel() { const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 5c575b93d4..b5f2fb6937 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -41,6 +41,9 @@ {{ 'version-control.export-entity-relations' | translate }} + + {{ 'version-control.export-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 9f998bc2af..2ff8c03227 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -62,7 +62,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], versionName: [null, [Validators.required]], - saveRelations: [false, []] + saveRelations: [false, []], + saveAttributes: [false, []] }); } @@ -78,7 +79,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni branch: this.createVersionFormGroup.get('branch').value, versionName: this.createVersionFormGroup.get('versionName').value, config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index cb71696b4e..e84beb03dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -29,6 +29,9 @@ {{ 'version-control.load-entity-relations' | translate }} + + {{ 'version-control.load-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 2cc615c011..ea5443e036 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -57,7 +57,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [false, []] + loadRelations: [false, []], + loadAttributes: [false, []] }); } @@ -73,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn versionId: this.versionId, externalEntityId: this.externalEntityId, config: { - loadRelations: this.restoreFormGroup.get('loadRelations').value + loadRelations: this.restoreFormGroup.get('loadRelations').value, + loadAttributes: this.restoreFormGroup.get('loadAttributes').value }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html new file mode 100644 index 0000000000..14ae91e2f2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html @@ -0,0 +1,41 @@ + +
+
+
+
+ + + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts new file mode 100644 index 0000000000..f569d96fac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts @@ -0,0 +1,66 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-remove-other-entities-confirm', + templateUrl: './remove-other-entities-confirm.component.html', + styleUrls: [] +}) +export class RemoveOtherEntitiesConfirmComponent extends PageComponent implements OnInit { + + @Input() + onClose: (result: boolean | null) => void; + + confirmFormGroup: FormGroup; + + removeOtherEntitiesConfirmText: SafeHtml; + + removeOtherEntitiesVerificationText = 'remove other entities'; + + constructor(protected store: Store, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + this.removeOtherEntitiesConfirmText = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.remove-other-entities-confirm-text')); + } + + ngOnInit(): void { + this.confirmFormGroup = this.fb.group({ + verification: [null, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + confirm(): void { + if (this.onClose) { + this.onClose(true); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html similarity index 78% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index b48162d06f..bdab6c061b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,24 +16,24 @@ -->
- +
- admin.git-repository-settings + admin.repository-settings -
+
-
+
admin.repository-url - + admin.repository-url-required @@ -46,12 +46,12 @@ admin.auth-method - - {{versionControlAuthMethodTranslations.get(method) | translate}} + + {{repositoryAuthMethodTranslations.get(method) | translate}} -
+
common.username
-
+
+ (fileNameChanged)="repositorySettingsForm.get('privateKeyFileName').patchValue($event)"> @@ -96,10 +96,10 @@ -
diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss index d1d55faf19..77ae0ffd8f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - mat-card.vc-settings { + mat-card.repository-settings { margin: 8px; } .fields-group { diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts new file mode 100644 index 0000000000..c73cf0e632 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -0,0 +1,216 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + RepositorySettings, + RepositoryAuthMethod, + repositoryAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; +import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { catchError, mergeMap, take } from 'rxjs/operators'; +import { of } from 'rxjs'; + +@Component({ + selector: 'tb-repository-settings', + templateUrl: './repository-settings.component.html', + styleUrls: ['./repository-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class RepositorySettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; + + repositorySettingsForm: FormGroup; + settings: RepositorySettings = null; + + repositoryAuthMethod = RepositoryAuthMethod; + repositoryAuthMethods = Object.values(RepositoryAuthMethod); + repositoryAuthMethodTranslations = repositoryAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.repositorySettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: ['main', []], + authMethod: [RepositoryAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.repositorySettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.repositorySettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.store.pipe( + select(selectHasRepository), + take(1), + mergeMap((hasRepository) => { + if (hasRepository) { + return this.adminService.getRepositorySettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + if (this.settings != null) { + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + } + }); + } + + checkAccess(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.checkRepositoryAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-repository-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.saveRepositorySettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: true })); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-repository-settings-title', ), + this.translate.instant('admin.delete-repository-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteRepositorySettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.repositorySettingsForm.reset({ defaultBranch: 'main', authMethod: RepositoryAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: false })); + } + ); + } + }); + } + + changePasswordChanged() { + if (this.changePassword) { + this.repositorySettingsForm.get('password').patchValue(''); + this.repositorySettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').patchValue(''); + this.repositorySettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: RepositoryAuthMethod = this.repositorySettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.repositorySettingsForm.get('privateKeyFileName').value; + if (authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.repositorySettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.repositorySettingsForm.get('password').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('password').disable({emitEvent}); + } + this.repositorySettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.repositorySettingsForm.get('privateKey').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.repositorySettingsForm.get('username').disable({emitEvent}); + this.repositorySettingsForm.get('password').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.repositorySettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.repositorySettingsForm.get('privateKey').clearValidators(); + } else { + this.repositorySettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.repositorySettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts deleted file mode 100644 index 3462fbb728..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ /dev/null @@ -1,218 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Input, OnInit } from '@angular/core'; -import { PageComponent } from '@shared/components/page.component'; -import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; -import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { select, Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { AdminService } from '@core/http/admin.service'; -import { - EntitiesVersionControlSettings, - VersionControlAuthMethod, - versionControlAuthMethodTranslationMap -} from '@shared/models/settings.models'; -import { ActionNotificationShow } from '@core/notification/notification.actions'; -import { TranslateService } from '@ngx-translate/core'; -import { isNotEmptyStr } from '@core/utils'; -import { DialogService } from '@core/services/dialog.service'; -import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; -import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; -import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap, take } from 'rxjs/operators'; -import { of } from 'rxjs'; - -@Component({ - selector: 'tb-version-control-settings', - templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] -}) -export class VersionControlSettingsComponent extends PageComponent implements OnInit { - - @Input() - detailsMode = false; - - versionControlSettingsForm: FormGroup; - settings: EntitiesVersionControlSettings = null; - - versionControlAuthMethod = VersionControlAuthMethod; - versionControlAuthMethods = Object.values(VersionControlAuthMethod); - versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; - - showChangePassword = false; - changePassword = false; - - showChangePrivateKeyPassword = false; - changePrivateKeyPassword = false; - - constructor(protected store: Store, - private adminService: AdminService, - private dialogService: DialogService, - private translate: TranslateService, - public fb: FormBuilder) { - super(store); - } - - ngOnInit() { - this.versionControlSettingsForm = this.fb.group({ - repositoryUri: [null, [Validators.required]], - defaultBranch: ['main', []], - authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], - username: [null, []], - password: [null, []], - privateKeyFileName: [null, [Validators.required]], - privateKey: [null, []], - privateKeyPassword: [null, []] - }); - this.updateValidators(false); - this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { - this.updateValidators(true); - }); - this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { - this.updateValidators(false); - }); - this.store.pipe( - select(selectHasVersionControl), - take(1), - mergeMap((hasVersionControl) => { - if (hasVersionControl) { - return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( - catchError(() => of(null)) - ); - } else { - return of(null); - } - }) - ).subscribe( - (settings) => { - this.settings = settings; - if (this.settings != null) { - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - } - }); - } - - checkAccess(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.checkVersionControlAccess(settings).subscribe(() => { - this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), - type: 'success' })); - }); - } - - save(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( - (savedSettings) => { - this.settings = savedSettings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - this.changePassword = false; - } else { - this.showChangePrivateKeyPassword = true; - this.changePrivateKeyPassword = false; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); - } - ); - } - - delete(formDirective: FormGroupDirective): void { - this.dialogService.confirm( - this.translate.instant('admin.delete-git-settings-title', ), - this.translate.instant('admin.delete-git-settings-text'), null, - this.translate.instant('action.delete') - ).subscribe((data) => { - if (data) { - this.adminService.deleteEntitiesVersionControlSettings().subscribe( - () => { - this.settings = null; - this.showChangePassword = false; - this.changePassword = false; - this.showChangePrivateKeyPassword = false; - this.changePrivateKeyPassword = false; - formDirective.resetForm(); - this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); - } - ); - } - }); - } - - changePasswordChanged() { - if (this.changePassword) { - this.versionControlSettingsForm.get('password').patchValue(''); - this.versionControlSettingsForm.get('password').markAsDirty(); - } - this.updateValidators(false); - } - - changePrivateKeyPasswordChanged() { - if (this.changePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); - this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); - } - this.updateValidators(false); - } - - updateValidators(emitEvent?: boolean): void { - const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; - const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; - if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.versionControlSettingsForm.get('username').enable({emitEvent}); - if (this.changePassword || !this.showChangePassword) { - this.versionControlSettingsForm.get('password').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('password').disable({emitEvent}); - } - this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } else { - this.versionControlSettingsForm.get('username').disable({emitEvent}); - this.versionControlSettingsForm.get('password').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); - if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } - if (isNotEmptyStr(privateKeyFileName)) { - this.versionControlSettingsForm.get('privateKey').clearValidators(); - } else { - this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); - } - } - this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); - } - -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index e5e650ac85..3cf10ab168 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -15,9 +15,9 @@ limitations under the License. --> - - + + (); - hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + hasRepository$ = this.store.pipe(select(selectHasRepository)); constructor(private store: Store) { @@ -61,7 +61,7 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index 958e7b1c9b..16baa3aa89 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -225,14 +225,14 @@ const routes: Routes = [ ] }, { - path: 'vc', - component: VersionControlAdminSettingsComponent, + path: 'repository', + component: RepositoryAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], - title: 'admin.git-settings', + title: 'admin.repository-settings', breadcrumb: { - label: 'admin.git-settings', + label: 'admin.repository-settings', icon: 'manage_history' } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 51612270df..85e3d04767 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version- HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlAdminSettingsComponent + RepositoryAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html similarity index 87% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html index 869067e1fb..7b408fc93a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html @@ -15,4 +15,4 @@ limitations under the License. --> - + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts similarity index 68% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts index a203f64426..1ee0a7288a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts @@ -20,16 +20,16 @@ import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormGroup } from '@angular/forms'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; @Component({ - selector: 'tb-version-control-admin-settings', - templateUrl: './version-control-admin-settings.component.html', + selector: 'tb-repository-admin-settings', + templateUrl: './repository-admin-settings.component.html', styleUrls: [] }) -export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class RepositoryAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { - @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + @ViewChild('repositorySettingsComponent') repositorySettingsComponent: RepositorySettingsComponent; constructor(protected store: Store) { super(store); @@ -39,6 +39,6 @@ export class VersionControlAdminSettingsComponent extends PageComponent implemen } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 6297809522..ff16183509 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -135,7 +135,7 @@ export const HelpLinks = { ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', queue: helpBaseUrl + '/docs/user-guide/queue', - versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' + repositorySettings: helpBaseUrl + '/docs/user-guide/ui/repository-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 8a1b316001..298281da11 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -16,6 +16,7 @@ import { ValidatorFn } from '@angular/forms'; import { isNotEmptyStr, isNumber } from '@core/utils'; +import { VersionCreateConfig } from '@shared/models/vc.models'; export const smtpPortPattern: RegExp = /^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/; @@ -397,23 +398,29 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid return smsProviderConfiguration; } -export enum VersionControlAuthMethod { +export enum RepositoryAuthMethod { USERNAME_PASSWORD = 'USERNAME_PASSWORD', PRIVATE_KEY = 'PRIVATE_KEY' } -export const versionControlAuthMethodTranslationMap = new Map([ - [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], - [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +export const repositoryAuthMethodTranslationMap = new Map([ + [RepositoryAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [RepositoryAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] ]); -export interface EntitiesVersionControlSettings { +export interface RepositorySettings { repositoryUri: string; defaultBranch: string; - authMethod: VersionControlAuthMethod; + authMethod: RepositoryAuthMethod; username: string; password: string; privateKeyFileName: string; privateKey: string; privateKeyPassword: string; } + +export interface AutoVersionCreateConfig extends VersionCreateConfig { + branch: string; +} + +export type AutoCommitSettings = {[entityType: string]: AutoVersionCreateConfig}; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0ad52d2e0b..cd1cba93ed 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -32,6 +32,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; + saveAttributes: boolean; } export enum VersionCreateRequestType { @@ -81,6 +82,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: res[entityType] = { syncStrategy: null, saveRelations: false, + saveAttributes: false, allEntities: true, entityIds: [] }; @@ -90,6 +92,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; + loadAttributes: boolean; } export enum VersionLoadRequestType { @@ -111,6 +114,7 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { removeOtherEntities: boolean; + findExistingEntityByName: boolean; } export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { @@ -123,7 +127,9 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En for (const entityType of exportableEntityTypes) { res[entityType] = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; } return res; @@ -176,18 +182,5 @@ export interface EntityDataDiff { } export function entityExportDataToJsonString(data: EntityExportData): string { - if (!data.relations) { - data.relations = []; - } - const allKeys = new Set(); - JSON.stringify(data, (key, value) => (allKeys.add(key), value)); - return JSON.stringify(data, Array.from(allKeys).sort((key1, key2) => { - if (key1 === 'relations') { - return 1; - } else if (key2 === 'relations') { - return -1; - } else { - return 0; - } - }), 4); + return JSON.stringify(data, null, 4); } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 4a4e228cc2..8a2f63cb8f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -58,7 +58,8 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", - "restore": "Restore" + "restore": "Restore", + "confirm": "Confirm" }, "aggregation": { "aggregation": "Aggregation", @@ -321,8 +322,7 @@ "queue-submit-strategy": "Submit strategy", "queue-processing-strategy": "Processing strategy", "queue-configuration": "Queue configuration", - "git-settings": "Git settings", - "git-repository-settings": "Git repository settings", + "repository-settings": "Repository settings", "repository-url": "Repository URL", "repository-url-required": "Repository URL is required.", "default-branch": "Default branch name", @@ -338,9 +338,9 @@ "enter-passphrase": "Enter passphrase", "change-passphrase": "Change passphrase", "check-access": "Check access", - "check-vc-access-success": "Git repository access successfully verified!", - "delete-git-settings-title": "Are you sure you want to delete git settings?", - "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + "check-repository-access-success": "Repository access successfully verified!", + "delete-repository-settings-title": "Are you sure you want to delete repository settings?", + "delete-repository-settings-text": "Be careful, after the confirmation the repository settings will be removed and version control feature will be unavailable." }, "alarm": { "alarm": "Alarm", @@ -3121,6 +3121,7 @@ "version-name-required": "Version name is required", "author": "Author", "export-entity-relations": "Export entity relations", + "export-entity-attributes": "Export entity attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3133,6 +3134,7 @@ "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations", + "load-entity-attributes": "Load entity attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3151,12 +3153,15 @@ "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "load-entities-relations": "Load entities relations", + "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", + "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", "no-entities-restored": "No entities restored", "created": "{{created}} created", "updated": "{{updated}} updated", - "deleted": "{{deleted}} deleted" + "deleted": "{{deleted}} deleted", + "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm." }, "widget": { "widget-library": "Widgets Library", From 59325226ac5020227644afee6a2641bec462e8b2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 16:38:10 +0300 Subject: [PATCH 139/262] Sort order and diff fixes --- .../DefaultEntitiesVersionControlService.java | 19 +++++++++-------- .../DefaultGitVersionControlQueueService.java | 4 ++-- .../common/data/sync/ie/EntityExportData.java | 21 +++++++++++++++++++ .../thingsboard/common/util/JacksonUtil.java | 10 ++++++--- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index fb68967661..05fbef899b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -328,16 +328,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityId externalId = ((ExportableEntity) entity).getExternalId(); if (externalId == null) externalId = entityId; - EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() - .exportRelations(true) - .exportAttributes(true) - .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), - otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion), - JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { - return new EntityDataDiff(currentVersion, otherVersion, rawDiff); - }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + otherVersion -> { + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(otherVersion.getRelations() != null) + .exportAttributes(otherVersion.getAttributes() != null) + .build()); + return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), + rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7019d6adb5..095c56f143 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -109,7 +109,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson = JacksonUtil.toPrettyString(entityData); + String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -130,7 +130,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu buildCommitRequest(commit).setDeleteMsg( TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() ).build() - ).build(), wrap(commit.getFuture(), null)); + ).build(), wrap(future, null)); return future; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 542e675940..0fbdae756e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -41,6 +43,15 @@ import java.util.Map; @Data public class EntityExportData> { + public static final Comparator relationsComparator = Comparator + .comparing(EntityRelation::getFrom, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTo, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTypeGroup) + .thenComparing(EntityRelation::getType); + + public static final Comparator attrComparator = Comparator + .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + @JsonTbEntity private E entity; private EntityType entityType; @@ -48,4 +59,14 @@ public class EntityExportData> { private List relations; private Map> attributes; + public EntityExportData sort() { + if (relations != null && !relations.isEmpty()) { + relations.sort(relationsComparator); + } + if (attributes != null && !attributes.isEmpty()) { + attributes.values().forEach(list -> list.sort(attrComparator)); + } + return this; + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index b6e2b54479..d622801814 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -18,8 +18,10 @@ package org.thingsboard.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; @@ -31,8 +33,10 @@ import java.util.Arrays; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() - .enable(SerializationFeature.INDENT_OUTPUT); + public static final ObjectMapper PRETTY_SORTED_JSON_MAPPER = JsonMapper.builder().enable(SerializationFeature.INDENT_OUTPUT) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -99,7 +103,7 @@ public class JacksonUtil { public static String toPrettyString(Object o) { try { - return PRETTY_JSON_MAPPER.writeValueAsString(o); + return PRETTY_SORTED_JSON_MAPPER.writeValueAsString(o); } catch (JsonProcessingException e) { throw new RuntimeException(e); } From 7bd51cb4cec451b38e0d348f61ed046a4b781e14 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:13:50 +0300 Subject: [PATCH 140/262] Public Customer import fix. No commit when no changes --- .../importing/impl/CustomerImportService.java | 6 +++++- .../DefaultGitVersionControlQueueService.java | 18 ++++++++++-------- .../data/sync/vc/RepositorySettings.java | 1 - .../DefaultClusterVersionControlService.java | 18 +++++++++++------- .../sync/vc/DefaultGitRepositoryService.java | 9 +++++---- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index 2a7570e644..aec96b2e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -42,7 +42,11 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return customerService.saveCustomer(customer); + if (customer.isPublic()) { + return customerService.findOrCreatePublicCustomer(tenantId); + } else { + return customerService.saveCustomer(customer); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 095c56f143..c87789a096 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -152,7 +152,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch), + .setBranchName(branch), pageLink ).build()); } @@ -162,8 +162,8 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityType.name()), + .setBranchName(branch) + .setEntityType(entityType.name()), pageLink ).build()); } @@ -173,10 +173,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), pageLink ).build()); } @@ -354,7 +354,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + if (commitResponse.getTs() > 0) { + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + } commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index eb1a97ff6c..50e23a1dba 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -21,7 +21,6 @@ import lombok.Data; import java.io.Serializable; @Data -@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 6ff7347be9..b28618372e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -457,14 +457,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { - reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() - .setTs(result.getVersion().getTimestamp()) - .setCommitId(result.getVersion().getId()) - .setName(result.getVersion().getName()) - .setAuthor(result.getVersion().getAuthor()) - .setAdded(result.getAdded()) + var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded()) .setModified(result.getModified()) - .setRemoved(result.getRemoved()))); + .setRemoved(result.getRemoved()); + + if (result.getVersion() != null) { + responseBuilder.setTs(result.getVersion().getTimestamp()) + .setCommitId(result.getVersion().getId()) + .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()); + } + + reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(responseBuilder)); } private void reply(VersionControlRequestCtx ctx, Optional e) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 6db0a49d4f..99dbebc3c9 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,10 +117,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); - repository.push(commit.getWorkingBranch(), commit.getBranch()); - - result.setVersion(toVersion(gitCommit)); + if (result.getAdded() > 0 || result.getModified() > 0 || result.getRemoved() > 0) { + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); + repository.push(commit.getWorkingBranch(), commit.getBranch()); + result.setVersion(toVersion(gitCommit)); + } return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; From 1610222b72528ae7399bfc82167210f9171d5764 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 17:27:53 +0300 Subject: [PATCH 141/262] Export/import api rate limiting --- .../apiusage/DefaultRateLimitService.java | 76 +++++++++++++++++++ .../service/apiusage/RateLimitService.java | 26 +++++++ .../DefaultEntitiesExportImportService.java | 11 +++ .../DefaultTenantProfileConfiguration.java | 5 +- .../server/common/msg/tools/TbRateLimits.java | 8 +- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 777e84f348..859f3b9da2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.apiusage.RateLimitService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; @@ -55,6 +57,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); + private final RateLimitService rateLimitService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE @@ -63,6 +67,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); @@ -73,6 +81,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 15eb429805..9953f7a3aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index afcbe1b3b9..9ca13b295d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -27,6 +27,7 @@ import java.time.Duration; */ public class TbRateLimits { private final LocalBucket bucket; + private final String configuration; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); @@ -42,8 +43,7 @@ public class TbRateLimits { } else { throw new IllegalArgumentException("Failed to parse rate limits configuration: " + limitsConfiguration); } - - + this.configuration = limitsConfiguration; } public boolean tryConsume() { @@ -54,4 +54,8 @@ public class TbRateLimits { return bucket.tryConsume(number); } + public String getConfiguration() { + return configuration; + } + } From ac0b1a98794b05ff713ed45568d5eca94a1cdfa9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:46:23 +0300 Subject: [PATCH 142/262] getEntityDataInfo API call --- .../EntitiesVersionControlController.java | 13 +++++++++ .../DefaultEntitiesVersionControlService.java | 8 ++++++ .../vc/EntitiesVersionControlService.java | 3 ++ .../common/data/sync/vc/EntityDataInfo.java | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bf7011867d..36aa6fd961 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -237,6 +238,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/info/{versionId}/{entityType}/{internalEntityUuid}") + public DeferredResult getEntityDataInfo(@PathVariable String versionId, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") public DeferredResult compareEntityDataToVersion(@PathVariable String branch, @PathVariable EntityType entityType, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 05fbef899b..cca21929c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -341,6 +342,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { + return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + } + + @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { return gitServiceQueue.listBranches(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 447ae1d021..88c23616b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -62,4 +63,6 @@ public interface EntitiesVersionControlService { ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; + + ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java new file mode 100644 index 0000000000..ab548b1cfc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityDataInfo { + boolean hasRelations; + boolean hasAttributes; +} From 9eb32add2bf5ebb330d14bfd23ce74d9f30df6cd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 18:11:24 +0300 Subject: [PATCH 143/262] UI for export/import api rate limiting --- .../default-tenant-profile-configuration.component.html | 8 ++++++++ .../default-tenant-profile-configuration.component.ts | 2 ++ ui-ngx/src/app/shared/models/tenant.model.ts | 3 +++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ 4 files changed, 15 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index fef4d4cef5..08c8df06a0 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -280,4 +280,12 @@ tenant-profile.transport-device-telemetry-data-points-rate-limit + + tenant-profile.tenant-entity-export-rate-limit + + + + tenant-profile.tenant-entity-import-rate-limit + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index ddab7f3aae..6f79c34279 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -67,6 +67,8 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA transportDeviceMsgRateLimit: [null, []], transportDeviceTelemetryMsgRateLimit: [null, []], transportDeviceTelemetryDataPointsRateLimit: [null, []], + tenantEntityExportRateLimit: [null, []], + tenantEntityImportRateLimit: [null, []], maxTransportMessages: [null, [Validators.required, Validators.min(0)]], maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], maxREExecutions: [null, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 33d1367c01..ea1ff3fdc5 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -41,6 +41,9 @@ export interface DefaultTenantProfileConfiguration { transportDeviceTelemetryMsgRateLimit?: string; transportDeviceTelemetryDataPointsRateLimit?: string; + tenantEntityExportRateLimit?: string; + tenantEntityImportRateLimit?: string; + maxTransportMessages: number; maxTransportDataPoints: number; maxREExecutions: number; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index db52c04ac1..90193b1499 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2944,6 +2944,8 @@ "transport-device-msg-rate-limit": "Transport device messages rate limit.", "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "tenant-entity-export-rate-limit": "Entity version creation rate limit", + "tenant-entity-import-rate-limit": "Entity version load rate limit", "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", "max-transport-messages-required": "Maximum number of transport messages is required.", "max-transport-messages-range": "Maximum number of transport messages can't be negative", From 7841c5ff6fa27ccbc80544d2ef222a63a0b3d19e Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 18:42:33 +0300 Subject: [PATCH 144/262] UI: Implement auto-commit settings --- .../server/controller/AdminController.java | 8 +- ui-ngx/src/app/core/http/admin.service.ts | 2 +- ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/components/home-components.module.ts | 7 +- .../vc/auto-commit-settings.component.html | 125 +++++++++++ .../vc/auto-commit-settings.component.scss | 74 ++++++ .../vc/auto-commit-settings.component.ts | 210 ++++++++++++++++++ ...entity-types-version-create.component.html | 4 +- .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 4 +- .../vc/entity-types-version-load.component.ts | 2 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../auto-commit-admin-settings.component.html | 23 ++ .../auto-commit-admin-settings.component.ts | 51 +++++ .../vc/branch-autocomplete.component.html | 4 +- .../vc/branch-autocomplete.component.ts | 3 + ui-ngx/src/app/shared/models/constants.ts | 3 +- .../assets/locale/locale.constant-en_US.json | 16 +- 19 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index ed740f73fb..470ec738af 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -273,7 +273,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings") + @GetMapping("/autoCommitSettings") @ResponseBody public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { try { @@ -287,7 +287,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings/exists") + @GetMapping("/autoCommitSettings/exists") @ResponseBody public Boolean autoCommitSettingsExists() throws ThingsboardException { try { @@ -301,7 +301,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vc/autoCommitSettings") + @PostMapping("/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); return autoCommitSettingsService.save(getTenantId(), settings); @@ -311,7 +311,7 @@ public class AdminController extends BaseController { notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) public void deleteAutoCommitSettings() throws ThingsboardException { try { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 38f18bd598..9b8c3b76b9 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -101,7 +101,7 @@ export class AdminService { return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); } - private autoCommitSettingsExists(config?: RequestConfig): Observable { + public autoCommitSettingsExists(config?: RequestConfig): Observable { return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 50938e5869..af62bf7918 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -369,7 +369,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '120px', + height: '160px', icon: 'settings', pages: [ { @@ -392,6 +392,13 @@ export class MenuService { type: 'link', path: '/settings/repository', icon: 'manage_history' + }, + { + id: guid(), + name: 'admin.auto-commit-settings', + type: 'link', + path: '/settings/auto-commit', + icon: 'settings_backup_restore' } ] } @@ -541,6 +548,11 @@ export class MenuService { name: 'admin.repository-settings', icon: 'manage_history', path: '/settings/repository', + }, + { + name: 'admin.auto-commit-settings', + icon: 'settings_backup_restore', + path: '/settings/auto-commit' } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 613f236cf6..153a8d9759 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -164,6 +164,7 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; +import { AutoCommitSettingsComponent } from '@home/components/vc/auto-commit-settings.component'; @NgModule({ declarations: @@ -298,7 +299,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], imports: [ CommonModule, @@ -427,7 +429,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html new file mode 100644 index 0000000000..3e668b54e6 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -0,0 +1,125 @@ + +
+ + +
+ admin.auto-commit-settings + +
+
+ + + +
+ + +
+ admin.auto-commit-entities +
+
+
+ + +
+ +
+
+
+ + + +
+ + +
+ +
+
+ + +
+ + +
+
+
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
+
+
+ + +
+
+
+ admin.no-auto-commit-entities-prompt +
+
+ + + +
+
+
+
+ + + +
+ + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss new file mode 100644 index 0000000000..ea8017b9f1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.auto-commit-settings { + margin: 8px; + .mat-divider { + position: relative; + } + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + tb-branch-autocomplete { + min-width: 200px; + max-width: 200px; + display: block; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts new file mode 100644 index 0000000000..8ac301338e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { AbstractControl, FormArray, FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { AutoCommitSettings, AutoVersionCreateConfig } from '@shared/models/settings.models'; +import { TranslateService } from '@ngx-translate/core'; +import { DialogService } from '@core/services/dialog.service'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-auto-commit-settings', + templateUrl: './auto-commit-settings.component.html', + styleUrls: ['./auto-commit-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class AutoCommitSettingsComponent extends PageComponent implements OnInit { + + autoCommitSettingsForm: FormGroup; + settings: AutoCommitSettings = null; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private sanitizer: DomSanitizer, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.autoCommitSettingsForm = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.adminService.autoCommitSettingsExists().pipe( + catchError(() => of(false)), + mergeMap((hasAutoCommitSettings) => { + if (hasAutoCommitSettings) { + return this.adminService.getAutoCommitSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(settings), {emitEvent: false}); + }); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.autoCommitSettingsForm.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.autoCommitSettingsForm.get('entityTypes') as FormArray).removeAt(index); + this.autoCommitSettingsForm.markAsDirty(); + } + + public addEnabled(): boolean { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + const config: AutoVersionCreateConfig = { + branch: null, + saveRelations: false, + saveAttributes: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + public removeAll() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + entityTypeText(entityTypeControl: AbstractControl): SafeHtml { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: AutoVersionCreateConfig = entityTypeControl.get('config').value; + let message = entityType ? this.translate.instant(entityTypeTranslations.get(entityType).typePlural) : 'Undefined'; + let branchName; + if (config.branch) { + branchName = config.branch; + } else { + branchName = this.translate.instant('version-control.default'); + } + message += ` (${this.translate.instant('version-control.auto-commit-to-branch', {branch: branchName})})`; + return this.sanitizer.bypassSecurityTrustHtml(message); + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + save(): void { + const value: [{entityType: string, config: AutoVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const settings: AutoCommitSettings = {}; + if (value && value.length) { + value.forEach((val) => { + settings[val.entityType] = val.config; + }); + } + this.adminService.saveAutoCommitSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(savedSettings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-auto-commit-settings-title', ), + this.translate.instant('admin.delete-auto-commit-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteAutoCommitSettings().subscribe( + () => { + this.settings = null; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(this.settings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + }); + } + + private prepareEntityTypesFormArray(settings: AutoCommitSettings | null): FormArray { + const entityTypesControls: Array = []; + if (settings) { + for (const entityType of Object.keys(settings)) { + const config = settings[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: AutoVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + branch: [config.branch, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []] + }) + } + ); + return entityTypeControl; + } + + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 3c1dd73117..d4da91028f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -17,7 +17,7 @@ -->
- version-control.entity-types + version-control.entities-to-export
version-control.no-entity-types + class="tb-prompt">version-control.no-entities-to-export-prompt
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 2ff8c03227..cc5ddcb568 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -41,6 +41,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Input() + entityName: string; + @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; @@ -61,7 +64,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ngOnInit(): void { this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], - versionName: [null, [Validators.required]], + versionName: [this.translate.instant('version-control.default-create-entity-version-name', + {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], saveAttributes: [false, []] }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts index 6036cf35a0..c72cbc0848 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -308,7 +308,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: this.versionName, @@ -322,6 +322,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, } } }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; } } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..dd851e9f23 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -15,27 +15,28 @@ limitations under the License. --> -
- +
+

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

 + *ngIf="entityDataInfo && (isLoading$ | async)"> -
+ +
- + {{ 'version-control.load-entity-relations' | translate }} - + {{ 'version-control.load-entity-attributes' | translate }}
-
+
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index d4da91028f..0ab5eca4e0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -66,10 +66,10 @@
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index d5a08ee141..c07fa138be 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -65,10 +65,10 @@
- {{ 'version-control.load-entities-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entities-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index b5f2fb6937..fa2855a6ed 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -39,10 +39,10 @@ - {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..0d97823c21 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,10 +27,10 @@
- {{ 'version-control.load-entity-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entity-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 40d427df24..2f428f11fe 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3125,8 +3125,8 @@ "version-name": "Version name", "version-name-required": "Version name is required", "author": "Author", - "export-entity-relations": "Export entity relations", - "export-entity-attributes": "Export entity attributes", + "export-relations": "Export relations", + "export-attributes": "Export attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3138,8 +3138,8 @@ "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", - "load-entity-relations": "Load entity relations", - "load-entity-attributes": "Load entity attributes", + "load-relations": "Load relations", + "load-attributes": "Load attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3159,8 +3159,6 @@ "add-entity-type": "Add entity type", "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", - "load-entities-relations": "Load entities relations", - "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", From 8382850d1a0c056561e6b071477f5b273a9cdee8 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 16:35:20 +0300 Subject: [PATCH 147/262] UI: Minor improvements --- .../vc/auto-commit-settings.component.html | 6 +++--- .../vc/auto-commit-settings.component.ts | 4 ++-- .../vc/complex-version-create.component.html | 3 ++- .../vc/complex-version-create.component.ts | 4 +++- .../entity-types-version-create.component.html | 6 +++--- .../vc/entity-types-version-create.component.ts | 4 ++-- .../vc/entity-types-version-load.component.html | 6 +++--- .../vc/entity-types-version-load.component.ts | 6 +++--- .../vc/entity-version-create.component.html | 6 +++--- .../vc/entity-version-create.component.ts | 2 +- .../vc/entity-version-restore.component.html | 6 +++--- .../vc/entity-version-restore.component.ts | 4 ++-- .../app/shared/components/popover.component.ts | 12 +++++++----- .../app/shared/components/popover.service.ts | 8 ++------ ui-ngx/src/app/shared/models/vc.models.ts | 17 ++++++++++++----- .../assets/locale/locale.constant-en_US.json | 4 +++- 16 files changed, 54 insertions(+), 44 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index a33d9ea374..e95cfc59fb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -75,12 +75,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index 8ac301338e..dc009d49cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -97,8 +97,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; const config: AutoVersionCreateConfig = { branch: null, - saveRelations: false, - saveAttributes: false + saveAttributes: true, + saveRelations: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 2c2f5bf274..4ace5e8849 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -39,13 +39,14 @@ {{ 'version-control.version-name-required' | translate }} - + version-control.default-sync-strategy {{syncStrategyTranslations.get(strategy) | translate}} + diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts index c152e6b447..c5c4124627 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -20,7 +20,7 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { ComplexVersionCreateRequest, createDefaultEntityTypesVersionCreate, - SyncStrategy, syncStrategyTranslationMap, + SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -51,6 +51,8 @@ export class ComplexVersionCreateComponent extends PageComponent implements OnIn syncStrategyTranslations = syncStrategyTranslationMap; + syncStrategyHints = syncStrategyHintMap; + resultMessage: string; versionCreateResult: VersionCreationResult = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 0ab5eca4e0..90712727e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -65,12 +65,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 44cb11dfa1..0d80b71adb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -183,8 +183,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionCreateConfig = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index c07fa138be..4c1cd26f84 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -64,12 +64,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 14586a5ee6..d983920cec 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -164,10 +164,10 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On public addEntityType() { const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 72dc6e7d48..90f338ad0e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -38,12 +38,12 @@ {{ 'version-control.version-name-required' | translate }} - - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index cc5ddcb568..c2c6fa48b5 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -67,7 +67,7 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni versionName: [this.translate.instant('version-control.default-create-entity-version-name', {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], - saveAttributes: [false, []] + saveAttributes: [true, []] }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 6a39e272e3..8dc81e38e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,12 +27,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 5e3a63243d..146a8570c2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -70,8 +70,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [true, []], - loadAttributes: [true, []] + loadAttributes: [true, []], + loadRelations: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; diff --git a/ui-ngx/src/app/shared/components/popover.component.ts b/ui-ngx/src/app/shared/components/popover.component.ts index c7e498a236..99321974af 100644 --- a/ui-ngx/src/app/shared/components/popover.component.ts +++ b/ui-ngx/src/app/shared/components/popover.component.ts @@ -55,7 +55,7 @@ import { POSITION_MAP, PropertyMapping } from '@shared/components/popover.models'; -import { distinctUntilChanged, takeUntil } from 'rxjs/operators'; +import { distinctUntilChanged, take, takeUntil } from 'rxjs/operators'; import { isNotEmptyStr, onParentScrollOrWindowResize } from '@core/utils'; export type TbPopoverTrigger = 'click' | 'focus' | 'hover' | null; @@ -372,7 +372,7 @@ export class TbPopoverComponent implements OnDestroy, OnInit { } get tbVisible(): boolean { - return this.visible; + return this.visible && this.tbAnimationState === 'active'; } visible = false; @@ -514,10 +514,12 @@ export class TbPopoverComponent implements OnDestroy, OnInit { const el = this.origin.elementRef.nativeElement; this.intersectionObserver.unobserve(el); } - - this.tbVisible = false; - this.tbVisibleChange.next(false); + this.tbAnimationState = 'void'; this.cdr.detectChanges(); + this.tbAnimationDone.pipe(take(1)).subscribe(() => { + this.tbVisible = false; + this.cdr.detectChanges(); + }); } updateByDirective(): void { diff --git a/ui-ngx/src/app/shared/components/popover.service.ts b/ui-ngx/src/app/shared/components/popover.service.ts index 2b64ee8009..713e8d337b 100644 --- a/ui-ngx/src/app/shared/components/popover.service.ts +++ b/ui-ngx/src/app/shared/components/popover.service.ts @@ -80,9 +80,7 @@ export class TbPopoverService { component.tbShowCloseButton = showCloseButton; component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { @@ -132,9 +130,7 @@ export class TbPopoverService { component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { visibleFn(false); - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index e3a7be5c54..ee718c3aec 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -64,6 +64,13 @@ export const syncStrategyTranslationMap = new Map( ] ); +export const syncStrategyHintMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge-hint'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite-hint'] + ] +); + export interface EntityTypeVersionCreateConfig extends VersionCreateConfig { syncStrategy: SyncStrategy; entityIds: string[]; @@ -81,8 +88,8 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: for (const entityType of exportableEntityTypes) { res[entityType] = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; @@ -126,10 +133,10 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; for (const entityType of exportableEntityTypes) { res[entityType] = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; } return res; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2bc8d6fdbe..ad9ba856ab 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3168,7 +3168,9 @@ "deleted": "{{deleted}} deleted", "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm.", "auto-commit-to-branch": "auto-commit to {{ branch }} branch", - "default-create-entity-version-name": "{{entityName}} update" + "default-create-entity-version-name": "{{entityName}} update", + "sync-strategy-merge-hint": "Sync strategy merge hint", + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" }, "widget": { "widget-library": "Widgets Library", From 54e388433479d7e8259bb05d5151fe85196a6c3c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 1 Jun 2022 16:55:38 +0300 Subject: [PATCH 148/262] Version load error and 'exportCredentials' setting support --- .../EntitiesVersionControlController.java | 3 +- .../impl/BaseEntityExportService.java | 4 +- .../exporting/impl/DeviceExportService.java | 7 +- .../impl/RuleChainExportService.java | 3 +- .../ie/importing/impl/AssetImportService.java | 3 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 3 +- .../impl/DashboardImportService.java | 2 +- .../importing/impl/DeviceImportService.java | 5 +- .../impl/DeviceProfileImportService.java | 3 +- .../impl/ImportServiceException.java | 20 ++ .../impl/MissingEntityException.java | 30 +++ .../impl/RuleChainImportService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 251 ++++++++++-------- .../DefaultGitVersionControlQueueService.java | 6 +- .../vc/EntitiesVersionControlService.java | 5 +- .../service/sync/vc/LoadEntityException.java | 32 +++ .../common/data/sync/ie/EntityExportData.java | 6 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../common/data/sync/vc/EntityDataInfo.java | 1 + .../common/data/sync/vc/EntityLoadError.java | 42 +++ .../data/sync/vc/EntityTypeLoadResult.java | 33 +++ .../data/sync/vc/VersionLoadResult.java | 29 +- .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 26 files changed, 368 insertions(+), 138 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 58c2659c4a..d55fe3dec0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -296,7 +297,7 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 80850cb462..9e14531e48 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -30,11 +30,11 @@ public abstract class BaseEntityExportService exportData, IdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index caa1e20ad7..ea938c89d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -64,7 +64,8 @@ import java.util.stream.Collectors; @Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { - @Autowired @Lazy + @Autowired + @Lazy private ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; @@ -94,7 +95,7 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -108,7 +109,7 @@ public abstract class BaseEntityImportService importResult, D exportData, @@ -210,7 +211,8 @@ public abstract class BaseEntityImportService() { @Override - public void onSuccess(@Nullable Void unused) {} + public void onSuccess(@Nullable Void unused) { + } @Override public void onFailure(Throwable thr) { @@ -246,7 +248,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) - .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); + .orElseThrow(() -> new MissingEntityException(externalId)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index aec96b2e47..2b2e2d1ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,7 +42,7 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { if (customer.isPublic()) { return customerService.findOrCreatePublicCustomer(tenantId); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 16ebdd580e..a67a536ded 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -64,7 +64,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { JsonNode configuration = dashboard.getConfiguration(); String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> { return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index eb123a63af..9a3730bba9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,11 +42,11 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java new file mode 100644 index 0000000000..1e869429d8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +public class ImportServiceException extends RuntimeException{ + private static final long serialVersionUID = -4932715239522125041L; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java new file mode 100644 index 0000000000..a0a961bfef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +public class MissingEntityException extends ImportServiceException { + + private static final long serialVersionUID = 3669135386955906022L; + @Getter + private final EntityId entityId; + + public MissingEntityException(EntityId entityId) { + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 1f1f15a74e..cd29be99d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -62,7 +62,7 @@ public class RuleChainImportService extends BaseEntityImportService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 735dc33a24..8aa4915fa8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,8 +22,10 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -45,10 +47,12 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityLoadError; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -64,12 +68,14 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -83,8 +89,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -172,6 +180,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -203,123 +212,157 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return Futures.transform(future, entityData -> { - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(false) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); - }, executor); + return Futures.transform(future, entityData -> doInTemplate(status -> loadSingleEntity(user, config, entityData)), executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return executor.submit(() -> transactionTemplate.execute(status -> { - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - List saveReferencesCallbacks = new ArrayList<>(); - List sendEventsCallbacks = new ArrayList<>(); - - versionLoadRequest.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()) - .forEach(entityType -> { - EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); + return executor.submit(() -> doInTemplate(status -> loadMultipleEntities(user, versionLoadRequest))); + } + default: + throw new IllegalArgumentException("Unsupported version load request"); + } + } - try { - int limit = 100; - int offset = 0; - List entityDataList; - do { - entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); - for (EntityExportData entityData : entityDataList) { - EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - offset += limit; - importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); - } while (entityDataList.size() == limit); - } catch (Exception e) { - throw new RuntimeException(e); - } - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created(created.get()) - .updated(updated.get()) - .build()); - }); + private VersionLoadResult doInTemplate(TransactionCallback result) { + try { + return transactionTemplate.execute(result); + } catch (LoadEntityException e) { + return onError(e.getData(), e.getCause()); + } + } - versionLoadRequest.getEntityTypes().keySet().stream() - .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) - .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) - .forEach(entityType -> { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - - sendEventsCallbacks.add(() -> { - entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), - entity, null, ActionType.DELETED, null, user); - }); - VersionLoadResult result = results.get(entityType); - result.setDeleted(result.getDeleted() + 1); - } - }); - }); + private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, + EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(false) + .build(), true, true); + return VersionLoadResult.success(EntityTypeLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + } - for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { - try { - saveReferencesCallback.run(); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } + private VersionLoadResult loadMultipleEntities(SecurityUser user, EntityTypeVersionLoadRequest versionLoadRequest) { + Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + + List entityTypes = versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + for (EntityType entityType : entityTypes) { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), versionLoadRequest.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); } - for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + results.put(entityType, EntityTypeLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); + } + + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); + EntityTypeLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); } - } - return new ArrayList<>(results.values()); - })); + }); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return VersionLoadResult.success(new ArrayList<>(results.values())); + } + + private VersionLoadResult onError(EntityExportData entityData, Throwable e) { + return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e)); + } + + private Optional analyze(Throwable e, EntityExportData entityData) { + if (e == null) { + return Optional.empty(); + } else { + if (e instanceof DeviceCredentialsValidationException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + } else if (e instanceof MissingEntityException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + } else { + return analyze(e.getCause(), entityData); } - default: - throw new IllegalArgumentException("Unsupported version load request"); } } @@ -347,7 +390,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), - entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null, false), MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index e168312206..759bc83acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -108,7 +108,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { SettableFuture future = SettableFuture.create(); - String path = getRelativePath(entityData.getEntityType(), getExternalId(entityData.getEntity())); + String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( @@ -120,10 +120,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return future; } - private EntityId getExternalId(ExportableEntity entity) { - return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); - } - @Override public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { SettableFuture future = SettableFuture.create(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 3b38182075..0a0f56eac6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -23,11 +23,12 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -49,7 +50,7 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java new file mode 100644 index 0000000000..a1b036d37b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@SuppressWarnings("rawtypes") +public class LoadEntityException extends RuntimeException { + + private static final long serialVersionUID = -1749719992370409504L; + @Getter + private final EntityExportData data; + + public LoadEntityException(EntityExportData data, Throwable cause) { + super(cause); + this.data = data; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 1109d9ef09..d715ec9611 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.ie; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -75,4 +76,9 @@ public class EntityExportData> { return this; } + @JsonIgnore + public EntityId getExternalId() { + return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0800a1f7c1..1a625640e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityExportSettings { private boolean exportRelations; private boolean exportAttributes; + private boolean exportCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 564b0134e3..9b95b8eca2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -28,4 +28,5 @@ public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java index ab548b1cfc..2529ec8af6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -25,4 +25,5 @@ import lombok.NoArgsConstructor; public class EntityDataInfo { boolean hasRelations; boolean hasAttributes; + boolean hasCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java new file mode 100644 index 0000000000..900a05e470 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EntityLoadError { + + private String type; + private EntityId source; + private EntityId target; + + public static EntityLoadError credentialsError(EntityId sourceId) { + return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build(); + } + + public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) { + return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java new file mode 100644 index 0000000000..cc0a8a642a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityTypeLoadResult { + private EntityType entityType; + private int created; + private int updated; + private int deleted; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index 0039546fb0..736de26079 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -15,19 +15,30 @@ */ package org.thingsboard.server.common.data.sync.vc; -import lombok.AllArgsConstructor; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.EntityType; + +import java.util.List; @Data -@AllArgsConstructor -@NoArgsConstructor @Builder +@JsonInclude(JsonInclude.Include.NON_NULL) public class VersionLoadResult { - private EntityType entityType; - private int created; - private int updated; - private int deleted; + + private List result; + private EntityLoadError error; + + public static VersionLoadResult success(List result) { + return VersionLoadResult.builder().result(result).build(); + } + + public static VersionLoadResult success(EntityTypeLoadResult result) { + return VersionLoadResult.builder().result(List.of(result)).build(); + } + + public static VersionLoadResult error(EntityLoadError error) { + return VersionLoadResult.builder().error(error).build(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 4426717fe9..86154bdfa0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -25,4 +25,5 @@ public class VersionCreateConfig implements Serializable { private boolean saveRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 2d68a2b7e3..a27cf06538 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -22,5 +22,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; + private boolean loadCredentials; } From 14023e4cc9fc1a957d8c8a9f81e06d14ea19aed0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 18:02:52 +0300 Subject: [PATCH 149/262] UI: Implement rule chain page version control. --- .../vc/entity-version-create.component.ts | 44 +++++++----- .../vc/entity-versions-table.component.html | 7 +- .../vc/entity-versions-table.component.ts | 9 ++- .../vc/version-control.component.html | 2 + .../vc/version-control.component.ts | 7 ++ .../rulechain/rulechain-page.component.html | 11 +++ .../rulechain/rulechain-page.component.scss | 15 ++++ .../rulechain/rulechain-page.component.ts | 68 +++++++++++++++++-- .../shared/components/breadcrumb.component.ts | 12 +++- 9 files changed, 148 insertions(+), 27 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index c2c6fa48b5..108d4aa081 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -27,6 +27,7 @@ import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; @Component({ selector: 'tb-entity-version-create', @@ -50,6 +51,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() onContentUpdated: () => void; + @Input() + onBeforeCreateVersion: () => Observable; + createVersionFormGroup: FormGroup; resultMessage: string; @@ -78,25 +82,29 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni } export(): void { - const request: SingleEntityVersionCreateRequest = { - entityId: this.entityId, - branch: this.createVersionFormGroup.get('branch').value, - versionName: this.createVersionFormGroup.get('versionName').value, - config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value, - saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (!result.added && !result.modified) { - this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); - if (this.onContentUpdated) { - this.onContentUpdated(); + const before = this.onBeforeCreateVersion ? this.onBeforeCreateVersion() : of(null); + before.subscribe(() => { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + config: { + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { + this.onClose(result, request.branch); } - } else if (this.onClose) { - this.onClose(result, request.branch); - } + }); }); } } + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index d2dbc119bf..79da474200 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -16,7 +16,7 @@ -->
-
+
@@ -45,6 +45,11 @@ update {{'version-control.create-entities-version' | translate }} +
+ + {{ 'version-control.export-credentials' | translate }} + {{ 'version-control.export-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index dc009d49cd..fd100f6247 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -39,6 +39,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit autoCommitSettingsForm: FormGroup; settings: AutoCommitSettings = null; + entityTypes = EntityType; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -98,7 +100,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const config: AutoVersionCreateConfig = { branch: null, saveAttributes: true, - saveRelations: false + saveRelations: false, + saveCredentials: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -199,7 +202,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit config: this.fb.group({ branch: [config.branch, []], saveRelations: [config.saveRelations, []], - saveAttributes: [config.saveAttributes, []] + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 4ace5e8849..9e15076788 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -70,7 +70,7 @@
-
{{ resultMessage }}
+
-
-
+
+
{{ 'version-control.no-entities-restored' | translate }}
-
{{ versionLoadResultMessage(versionLoadResult) }}
+
+
{{ entityTypeLoadResultMessage(entityTypeLoadResult) }}
-
{{ resultMessage }}
+
{{ resultMessage }}
- -
+
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + + +
+
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 146a8570c2..c31030f74b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -30,11 +30,12 @@ import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { delay } from 'rxjs/operators'; +import { SafeHtml } from '@angular/platform-browser'; @Component({ selector: 'tb-entity-version-restore', templateUrl: './entity-version-restore.component.html', - styleUrls: [] + styleUrls: ['./version-control.scss'] }) export class EntityVersionRestoreComponent extends PageComponent implements OnInit { @@ -51,7 +52,7 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: EntityId; @Input() - onClose: (result: Array | null) => void; + onClose: (result: VersionLoadResult | null) => void; @Input() popoverComponent: TbPopoverComponent; @@ -60,6 +61,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn restoreFormGroup: FormGroup; + errorMessage: SafeHtml; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -71,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], - loadRelations: [true, []] + loadRelations: [true, []], + loadCredentials: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -95,13 +99,22 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: this.externalEntityId, config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, - loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false + loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { - this.onClose(result); + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else { + if (this.onClose) { + this.onClose(result); + } } }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 79da474200..b6accc40dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -15,8 +15,8 @@ limitations under the License. --> -
-
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index ec5cb02256..3256c3c956 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -20,6 +20,13 @@ height: 100%; display: block; .tb-entity-table { + + &.tb-popover-mode { + position: relative; + width: 800px; + height: 600px; + } + .tb-entity-table-content { width: 100%; height: 100%; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 38b0a33da4..d776939aaf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -49,6 +49,7 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -63,7 +64,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni singleEntityMode = false; @Input() - popoverMode = false; + popoverComponent: TbPopoverComponent; @Input() onBeforeCreateVersion: () => Observable; @@ -211,14 +212,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - createVersionPopover.updatePosition(); - setTimeout(() => { - createVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + createVersionPopover.tbComponentRef.instance.popoverComponent = createVersionPopover; } } @@ -243,14 +239,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - complexCreateVersionPopover.updatePosition(); - setTimeout(() => { - complexCreateVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + complexCreateVersionPopover.tbComponentRef.instance.popoverComponent = complexCreateVersionPopover; } } @@ -293,9 +284,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni versionName: entityVersion.name, versionId: entityVersion.id, externalEntityId: this.externalEntityIdValue, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreVersionPopover.hide(); - if (result && result.length) { + if (result && !result.error && result.result.length) { this.versionRestored.emit(); } } @@ -318,10 +309,11 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni branch: this.branch, versionName: entityVersion.name, versionId: entityVersion.id, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreEntitiesVersionPopover.hide(); } }, {}, {}, {}, false); + restoreEntitiesVersionPopover.tbComponentRef.instance.popoverComponent = restoreEntitiesVersionPopover; } } diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index bdab6c061b..3e8bd271f3 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.repository-settings diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts index c73cf0e632..1120185db9 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit } from '@angular/core'; +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; import { select, Store } from '@ngrx/store'; @@ -33,6 +33,7 @@ import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; import { selectHasRepository } from '@core/auth/auth.selectors'; import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-repository-settings', @@ -44,6 +45,9 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit @Input() detailsMode = false; + @Input() + popoverComponent: TbPopoverComponent; + repositorySettingsForm: FormGroup; settings: RepositorySettings = null; @@ -61,6 +65,7 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit private adminService: AdminService, private dialogService: DialogService, private translate: TranslateService, + private cd: ChangeDetectorRef, public fb: FormBuilder) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 0ede98554d..eaf80fd664 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -16,11 +16,12 @@ --> { this.reloadRuleChain(); }); diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index ee718c3aec..0c6a667b88 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -33,6 +33,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; + saveCredentials: boolean; } export enum VersionCreateRequestType { @@ -90,6 +91,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: true, saveRelations: true, + saveCredentials: true, allEntities: true, entityIds: [] }; @@ -100,6 +102,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; + loadCredentials: boolean; } export enum VersionLoadRequestType { @@ -135,6 +138,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En res[entityType] = { loadAttributes: true, loadRelations: true, + loadCredentials: true, removeOtherEntities: false, findExistingEntityByName: true }; @@ -161,13 +165,36 @@ export interface VersionCreationResult { removed: number; } -export interface VersionLoadResult { +export interface EntityTypeLoadResult { entityType: EntityType; created: number; updated: number; deleted: number; } +export enum EntityLoadErrorType { + DEVICE_CREDENTIALS_CONFLICT = 'DEVICE_CREDENTIALS_CONFLICT', + MISSING_REFERENCED_ENTITY = 'MISSING_REFERENCED_ENTITY' +} + +export const entityLoadErrorTranslationMap = new Map( + [ + [EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT, 'version-control.device-credentials-conflict'], + [EntityLoadErrorType.MISSING_REFERENCED_ENTITY, 'version-control.missing-referenced-entity'] + ] +); + +export interface EntityLoadError { + type: EntityLoadErrorType; + source: EntityId; + target: EntityId; +} + +export interface VersionLoadResult { + result: Array; + error: EntityLoadError; +} + export interface EntityExportData> { entity: E; entityType: EntityType; @@ -195,4 +222,5 @@ export function entityExportDataToJsonString(data: EntityExportData): strin export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; + hasCredentials: boolean; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ad9ba856ab..8a4d8ff4db 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3127,6 +3127,7 @@ "author": "Author", "export-relations": "Export relations", "export-attributes": "Export attributes", + "export-credentials": "Export credentials", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3140,6 +3141,7 @@ "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-relations": "Load relations", "load-attributes": "Load attributes", + "load-credentials": "Load credentials", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3158,7 +3160,7 @@ "no-entities-to-restore-prompt": "Please specify entities to restore", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.
{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.
{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", @@ -3170,7 +3172,9 @@ "auto-commit-to-branch": "auto-commit to {{ branch }} branch", "default-create-entity-version-name": "{{entityName}} update", "sync-strategy-merge-hint": "Sync strategy merge hint", - "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint", + "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", + "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}." }, "widget": { "widget-library": "Widgets Library", From da224c9651f6ac14993776704b82aed6a2b319ee Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 2 Jun 2022 14:10:56 +0300 Subject: [PATCH 154/262] UI: Implement dashboard version control --- .../dashboard-page.component.html | 7 +++ .../dashboard-page.component.ts | 60 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index dd7212cf5b..553971005f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -81,6 +81,13 @@ (click)="isFullscreen = !isFullscreen"> {{ isFullscreen ? 'fullscreen_exit' : 'fullscreen' }} +
- + phone-input.phone-input-label + {{ 'phone-input.phone-input-required' | translate }} {{ 'phone-input.phone-input-validation' | translate }} - - {{ 'phone-input.phone-input-pattern' | translate }} -
diff --git a/ui-ngx/src/app/shared/components/phone-input.component.scss b/ui-ngx/src/app/shared/components/phone-input.component.scss index 0145a13d62..872c686af0 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.scss +++ b/ui-ngx/src/app/shared/components/phone-input.component.scss @@ -21,6 +21,7 @@ .phone-input { width: 100%; + max-width: 290px; } } diff --git a/ui-ngx/src/app/shared/components/phone-input.component.ts b/ui-ngx/src/app/shared/components/phone-input.component.ts index 536b1cc723..bbee8098f6 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.ts +++ b/ui-ngx/src/app/shared/components/phone-input.component.ts @@ -33,6 +33,7 @@ import examples from 'libphonenumber-js/examples.mobile.json'; import { CountryCode, getExampleNumber, parsePhoneNumberFromString } from 'libphonenumber-js'; import { phoneNumberPattern } from '@shared/models/settings.models'; import { Subscription } from 'rxjs'; +import { FloatLabelType, MatFormFieldAppearance } from '@angular/material/form-field/form-field'; @Component({ selector: 'tb-phone-input', @@ -58,15 +59,8 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida @Input() defaultCountry: CountryCode = 'US'; @Input() enableFlagsSelect = true; @Input() required = true; - - private floatLabel = 'always'; - get showLabel(): string { - return this.floatLabel; - } - @Input() - set showLabel(value) { - this.floatLabel = value ? 'always' : 'never'; - } + @Input() floatLabel: FloatLabelType = 'always'; + @Input() appearance: MatFormFieldAppearance = 'legacy'; allCountries: Array = []; phonePlaceholder: string; @@ -134,6 +128,8 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida focus() { const phoneNumber = this.phoneFormGroup.get('phoneNumber'); + this.phoneFormGroup.markAsPristine(); + this.phoneFormGroup.markAsUntouched(); if (!phoneNumber.value) { phoneNumber.patchValue(this.countryCallingCode); } @@ -220,7 +216,7 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida private updateModel() { const phoneNumber = this.phoneFormGroup.get('phoneNumber'); - if (phoneNumber.valid && this.modelValue !== phoneNumber.value) { + if (phoneNumber.valid && phoneNumber.value) { this.modelValue = phoneNumber.value; this.propagateChange(this.modelValue); } else { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b4bd223ed4..d8a08a5677 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4435,7 +4435,8 @@ "phone-input-label": "Phone number", "phone-input-required": "Phone number is required", "phone-input-validation": "Phone number is invalid or not possible", - "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. {{phoneNumber}}" + "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. {{phoneNumber}}", + "phone-input-hint": "Phone Number in E.164 format, ex. {{phoneNumber}}" }, "custom": { "widget-action": { From e958cb0fab98652c776725554780e32984f22102 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 7 Jun 2022 13:21:47 +0300 Subject: [PATCH 179/262] Tiny Refactoring --- .../DefaultEntitiesVersionControlService.java | 60 ++++++++++--------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c815c34a13..1a07b468e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -138,36 +138,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { - SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + handleSingleEntityRequest(user, commit, gitFutures, (SingleEntityVersionCreateRequest) request); break; } case COMPLEX: { - ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); - } - - if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) - , 100, entity -> { - try { - gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } else { - for (UUID entityId : config.getEntityIds()) { - try { - gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - }); + handleComplexRequest(user, commit, gitFutures, (ComplexVersionCreateRequest) request); break; } } @@ -175,6 +150,37 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, executor); } + private void handleSingleEntityRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, SingleEntityVersionCreateRequest versionCreateRequest) throws Exception { + gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + } + + private void handleComplexRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, ComplexVersionCreateRequest versionCreateRequest) { + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { + gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + } + + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + }); + } + private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) From 676a2b645ade72c30f537424397ac1b1f9e35344 Mon Sep 17 00:00:00 2001 From: kalutkaz Date: Tue, 7 Jun 2022 13:26:04 +0300 Subject: [PATCH 180/262] deleteMinWidth --- .../date-range-navigator-panel.component.scss | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator-panel.component.scss b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator-panel.component.scss index 036398d48e..9ed640d33d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/date-range-navigator/date-range-navigator-panel.component.scss @@ -29,7 +29,6 @@ .mat-padding { padding: 16px; - min-width: 560px; } - + } From fcd84f627770589407bfaf7c013957a3029cd919 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 7 Jun 2022 15:18:40 +0300 Subject: [PATCH 181/262] Export/import of Widgets Bundles --- .../main/data/upgrade/3.3.4/schema_update.sql | 2 + .../EntitiesVersionControlController.java | 2 +- .../DefaultEntitiesExportImportService.java | 3 +- .../DefaultExportableEntitiesService.java | 23 ----- .../impl/WidgetsBundleExportService.java | 60 ++++++++++++ .../impl/BaseEntityImportService.java | 3 + .../impl/WidgetsBundleImportService.java | 96 +++++++++++++++++++ .../server/common/data/sync/JsonTbEntity.java | 4 +- .../common/data/sync/ie/EntityExportData.java | 5 +- .../data/sync/ie/WidgetsBundleExportData.java | 42 ++++++++ .../common/data/widget/WidgetsBundle.java | 39 +++----- .../dao/model/sql/WidgetsBundleEntity.java | 9 ++ .../dao/sql/widget/JpaWidgetsBundleDao.java | 15 +++ .../sql/widget/WidgetsBundleRepository.java | 6 +- .../server/dao/widget/WidgetsBundleDao.java | 4 +- .../main/resources/sql/schema-entities.sql | 3 +- 16 files changed, 258 insertions(+), 58 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index 7fe6dd7ce1..babb630e9e 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -26,6 +26,8 @@ ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE customer ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE widgets_bundle + ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 337a850ab1..701719af58 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -310,7 +310,7 @@ public class EntitiesVersionControlController extends BaseController { public DeferredResult loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + accessControlService.checkPermission(user, Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index e9f90cf96b..76ec5f100d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -57,7 +57,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, - EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE + EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE, + EntityType.WIDGETS_BUNDLE ); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 82e5b9c60c..8434127aa5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -63,7 +63,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi private final Map> daos = new HashMap<>(); - private final EntityService entityService; private final AccessControlService accessControlService; @@ -141,28 +140,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return findEntitiesByQuery(tenantId, customerId, query); - } - - private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { - try { - return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() - .map(EntityData::getEntityId) - .collect(Collectors.toList()); - } catch (DataAccessException e) { - log.error("Failed to find entity data by query: {}", e.getMessage()); - throw new IllegalArgumentException("Entity filter cannot be processed"); - } - } - @Override public void deleteByTenantIdAndId(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java new file mode 100644 index 0000000000..bb7f5c08a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.List; +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class WidgetsBundleExportService extends BaseEntityExportService { + + private final WidgetTypeService widgetTypeService; + + @Override + protected void setRelatedEntities(TenantId tenantId, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, EntityExportSettings settings) { + if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { + throw new IllegalArgumentException("Export of system Widget Bundles is not allowed"); + } + + List widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(tenantId, widgetsBundle.getAlias()); + exportData.setWidgets(widgets); + } + + @Override + protected WidgetsBundleExportData newExportData() { + return new WidgetsBundleExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.WIDGETS_BUNDLE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 5221087a6b..3ba5bfcdbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; @@ -75,6 +76,8 @@ public abstract class BaseEntityImportService { + + private final WidgetsBundleService widgetsBundleService; + private final WidgetTypeService widgetTypeService; + + @Override + protected void setOwner(TenantId tenantId, WidgetsBundle widgetsBundle, IdProvider idProvider) { + widgetsBundle.setTenantId(tenantId); + } + + @Override + protected WidgetsBundle prepareAndSave(TenantId tenantId, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { + WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); + if (widgetsBundle.getId() == null) { + for (WidgetTypeDetails widget : exportData.getWidgets()) { + widget.setId(null); + widget.setTenantId(tenantId); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + } else { + Map existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(tenantId, savedWidgetsBundle.getAlias()).stream() + .collect(Collectors.toMap(BaseWidgetType::getAlias, w -> w)); + for (WidgetTypeDetails widget : exportData.getWidgets()) { + WidgetTypeInfo existingWidget; + if ((existingWidget = existingWidgets.remove(widget.getAlias())) != null) { + widget.setId(existingWidget.getId()); + widget.setCreatedTime(existingWidget.getCreatedTime()); + } else { + widget.setId(null); + } + widget.setTenantId(tenantId); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + existingWidgets.values().stream() + .map(BaseWidgetType::getId) + .forEach(widgetTypeId -> widgetTypeService.deleteWidgetType(tenantId, widgetTypeId)); + } + return savedWidgetsBundle; + } + + @Override + protected void onEntitySaved(SecurityUser user, WidgetsBundle savedWidgetsBundle, WidgetsBundle oldWidgetsBundle) throws ThingsboardException { + super.onEntitySaved(user, savedWidgetsBundle, oldWidgetsBundle); + entityNotificationService.notifySendMsgToEdgeService(user.getTenantId(), savedWidgetsBundle.getId(), + oldWidgetsBundle == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + } + + @Override + public EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index 9d326246d4..4bb636c73b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -41,7 +42,8 @@ import java.lang.annotation.Target; @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), @Type(name = "ASSET", value = Asset.class), @Type(name = "DASHBOARD", value = Dashboard.class), - @Type(name = "CUSTOMER", value = Customer.class) + @Type(name = "CUSTOMER", value = Customer.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundle.class) }) public @interface JsonTbEntity { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index d9f0ca4834..5283894739 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -31,7 +30,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -40,7 +38,8 @@ import java.util.Map; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), - @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundleExportData.class) }) @JsonInclude(JsonInclude.Include.NON_NULL) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java new file mode 100644 index 0000000000..aa7600a48d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.widget.BaseWidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.Comparator; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundleExportData extends EntityExportData { + + @JsonProperty(index = 3) + private List widgets; + + @Override + public EntityExportData sort() { + super.sort(); + widgets.sort(Comparator.comparing(BaseWidgetType::getAlias)); + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java index 1515aceb62..4f3de71512 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.common.data.widget; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBased; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +30,8 @@ import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class WidgetsBundle extends SearchTextBased implements HasTenantId { +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundle extends SearchTextBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -7627368878362410489L; @@ -63,6 +67,10 @@ public class WidgetsBundle extends SearchTextBased implements H @ApiModelProperty(position = 7, value = "Description", readOnly = true) private String description; + @Getter + @Setter + private WidgetsBundleId externalId; + public WidgetsBundle() { super(); } @@ -78,6 +86,7 @@ public class WidgetsBundle extends SearchTextBased implements H this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + this.externalId = widgetsBundle.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the Widget Bundle Id. " + @@ -100,31 +109,10 @@ public class WidgetsBundle extends SearchTextBased implements H return getTitle(); } + @JsonIgnore @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (tenantId != null ? tenantId.hashCode() : 0); - result = 31 * result + (alias != null ? alias.hashCode() : 0); - result = 31 * result + (title != null ? title.hashCode() : 0); - result = 31 * result + (image != null ? image.hashCode() : 0); - result = 31 * result + (description != null ? description.hashCode() : 0); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - WidgetsBundle that = (WidgetsBundle) o; - - if (tenantId != null ? !tenantId.equals(that.tenantId) : that.tenantId != null) return false; - if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false; - if (title != null ? !title.equals(that.title) : that.title != null) return false; - if (image != null ? !image.equals(that.image) : that.image != null) return false; - if (description != null ? !description.equals(that.description) : that.description != null) return false; - return true; + public String getName() { + return title; } @Override @@ -133,7 +121,6 @@ public class WidgetsBundle extends SearchTextBased implements H sb.append("tenantId=").append(tenantId); sb.append(", alias='").append(alias).append('\''); sb.append(", title='").append(title).append('\''); - sb.append(", image='").append(image).append('\''); sb.append(", description='").append(description).append('\''); sb.append('}'); return sb.toString(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 5887c6f61e..cbd6068d7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -54,6 +54,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl @Column(name = ModelConstants.WIDGETS_BUNDLE_DESCRIPTION) private String description; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public WidgetsBundleEntity() { super(); } @@ -70,6 +73,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + if (widgetsBundle.getExternalId() != null) { + this.externalId = widgetsBundle.getExternalId().getId(); + } } @Override @@ -93,6 +99,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl widgetsBundle.setTitle(title); widgetsBundle.setImage(image); widgetsBundle.setDescription(description); + if (externalId != null) { + widgetsBundle.setExternalId(new WidgetsBundleId(externalId)); + } return widgetsBundle; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 56dd4c0446..5f3445f038 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -93,4 +93,19 @@ public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findTenantWidgetsBundlesByTenantId(tenantId, pageLink); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 0de8d0220b..517a710cad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import java.util.UUID; @@ -27,7 +28,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 4/23/2017. */ -public interface WidgetsBundleRepository extends JpaRepository { +public interface WidgetsBundleRepository extends JpaRepository, ExportableEntityRepository { WidgetsBundleEntity findWidgetsBundleByTenantIdAndAlias(UUID tenantId, String alias); @@ -49,4 +50,7 @@ public interface WidgetsBundleRepository extends JpaRepository { +public interface WidgetsBundleDao extends Dao, ExportableEntityDao { /** * Save or update widgets bundle object diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index dd18da8722..e232e5deb2 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -423,7 +423,8 @@ CREATE TABLE IF NOT EXISTS widgets_bundle ( tenant_id uuid, title varchar(255), image varchar(1000000), - description varchar(255) + description varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS entity_view ( From bcfb7388c5730454b4991d251af3cdabfcb89c34 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 7 Jun 2022 15:47:15 +0300 Subject: [PATCH 182/262] Fix entity removing in vc --- .../DefaultEntitiesExportImportService.java | 1 - .../DefaultExportableEntitiesService.java | 65 +++++++++++++------ .../exporting/ExportableEntitiesService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 2 +- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 76ec5f100d..4e9e737076 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -47,7 +47,6 @@ import java.util.Map; @TbCoreComponent @RequiredArgsConstructor @Slf4j -@SuppressWarnings("rawtypes") public class DefaultEntitiesExportImportService implements EntitiesExportImportService { private final Map> exportServices = new HashMap<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 8434127aa5..9ea16477aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -18,28 +18,32 @@ package org.thingsboard.server.service.sync.ie.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.query.EntityData; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; -import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -47,13 +51,9 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; - -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; +import java.util.function.BiConsumer; @Service @TbCoreComponent @@ -62,6 +62,7 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME public class DefaultExportableEntitiesService implements ExportableEntitiesService { private final Map> daos = new HashMap<>(); + private final Map> removers = new HashMap<>(); private final AccessControlService accessControlService; @@ -141,14 +142,13 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi @Override - public void deleteByTenantIdAndId(TenantId tenantId, I id) { + public void removeById(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); - Dao dao = getDao(entityType); - if (dao == null) { + BiConsumer entityRemover = removers.get(entityType); + if (entityRemover == null) { throw new IllegalArgumentException("Unsupported entity type " + entityType); } - - dao.removeById(tenantId, id.getId()); + entityRemover.accept(tenantId, id); } @@ -182,4 +182,31 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi }); } + @Autowired + private void setRemovers(CustomerService customerService, AssetService assetService, RuleChainService ruleChainService, + DashboardService dashboardService, DeviceProfileService deviceProfileService, + DeviceService deviceService, WidgetsBundleService widgetsBundleService) { + removers.put(EntityType.CUSTOMER, (tenantId, entityId) -> { + customerService.deleteCustomer(tenantId, (CustomerId) entityId); + }); + removers.put(EntityType.ASSET, (tenantId, entityId) -> { + assetService.deleteAsset(tenantId, (AssetId) entityId); + }); + removers.put(EntityType.RULE_CHAIN, (tenantId, entityId) -> { + ruleChainService.deleteRuleChainById(tenantId, (RuleChainId) entityId); + }); + removers.put(EntityType.DASHBOARD, (tenantId, entityId) -> { + dashboardService.deleteDashboard(tenantId, (DashboardId) entityId); + }); + removers.put(EntityType.DEVICE_PROFILE, (tenantId, entityId) -> { + deviceProfileService.deleteDeviceProfile(tenantId, (DeviceProfileId) entityId); + }); + removers.put(EntityType.DEVICE, (tenantId, entityId) -> { + deviceService.deleteDevice(tenantId, (DeviceId) entityId); + }); + removers.put(EntityType.WIDGETS_BUNDLE, (tenantId, entityId) -> { + widgetsBundleService.deleteWidgetsBundle(tenantId, (WidgetsBundleId) entityId); + }); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 46b3b1b302..6ff2c96dcd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -38,7 +38,7 @@ public interface ExportableEntitiesService { , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); - void deleteByTenantIdAndId(TenantId tenantId, I id); + void removeById(TenantId tenantId, I id); void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 1a07b468e9..6737bca5e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -352,7 +352,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (ThingsboardException e) { throw new RuntimeException(e); } - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + exportableEntitiesService.removeById(user.getTenantId(), entity.getId()); sendEventsCallbacks.add(() -> { entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), From 850327e83b516cd172a2cb45e246c9e11f2eca52 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 8 Jun 2022 12:52:56 +0300 Subject: [PATCH 183/262] UI: Update version control models --- ui-ngx/src/app/shared/models/vc.models.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0c6a667b88..71aaa58f1f 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -195,10 +195,21 @@ export interface VersionLoadResult { error: EntityLoadError; } +export interface AttributeExportData { + key: string; + lastUpdateTs: number; + booleanValue: boolean; + strValue: string; + longValue: number; + doubleValue: number; + jsonValue: string; +} + export interface EntityExportData> { entity: E; entityType: EntityType; relations: Array; + attributes: {[key: string]: Array}; } export interface DeviceExportData extends EntityExportData { From 25f3cf640a3774ac7d5b62c05122c4a1e14d9c82 Mon Sep 17 00:00:00 2001 From: kalutkaz Date: Thu, 9 Jun 2022 13:25:19 +0300 Subject: [PATCH 184/262] updated scaling radius object --- .../home/components/widget/lib/flot-widget.ts | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts index fef3e487e9..3c93d03e84 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts @@ -312,7 +312,9 @@ export class TbFlot { if (this.settings.stroke) { this.options.series.pie.stroke.color = this.settings.stroke.color || '#fff'; this.options.series.pie.stroke.width = this.settings.stroke.width || 0; - this.scalingPieRadius(); + if (this.options.series.pie.stroke.width) { + this.scalingPieRadius(); + } } if (this.options.series.pie.label.show) { @@ -692,13 +694,10 @@ export class TbFlot { } private scalingPieRadius() { - // if (this.options.series.pie.stroke?.color !== '#fff' && this.options.series.pie.stroke?.width !== 0) { let scalingLine; this.ctx.width > this.ctx.height ? scalingLine = this.ctx.height : scalingLine = this.ctx.width; let changeRadius = this.options.series.pie.stroke.width / scalingLine; this.options.series.pie.radius = changeRadius < 1 ? this.settings.radius - changeRadius : 0; - console.log(this.options.series.pie.radius); - // } } public resize() { @@ -707,21 +706,21 @@ export class TbFlot { this.resizeTimeoutHandle = null; } if (this.plot && this.plotInited) { - const width = this.$element.width(); - const height = this.$element.height(); - // if (this.chartType === 'pie') { - // this.scalingPieRadius(); - // } - if (width && height) { - this.plot.resize(); - if (this.chartType !== 'pie') { - this.plot.setupGrid(); - } else { + if (this.chartType === 'pie' && this.settings.stroke?.width) { this.scalingPieRadius(); - } - this.plot.draw(); + this.redrawPlot(); } else { - this.resizeTimeoutHandle = setTimeout(this.resize.bind(this), 30); + const width = this.$element.width(); + const height = this.$element.height(); + if (width && height) { + this.plot.resize(); + if (this.chartType !== 'pie') { + this.plot.setupGrid(); + } + this.plot.draw(); + } else { + this.resizeTimeoutHandle = setTimeout(this.resize.bind(this), 30); + } } } } From 863200d6e7b7ddffa0a6e42feaf3dcddccb739f1 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Thu, 9 Jun 2022 18:02:31 +0300 Subject: [PATCH 185/262] UI: refactoring --- .../home/components/entity/entities-table.component.html | 2 +- .../home/components/entity/entities-table.component.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html index c3350c63a4..5f023b2efb 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html @@ -262,7 +262,7 @@ class="no-data-found">{{ 'common.loading' | translate }} - { - this.paginator.pageIndex = Number(params.page) || 0; - this.paginator.pageSize = Number(params.pageSize) || this.defaultPageSize; + if (this.displayPagination) { + this.paginator.pageIndex = Number(params.page) || 0; + this.paginator.pageSize = Number(params.pageSize) || this.defaultPageSize; + } this.sort.active = params.property || this.entitiesTableConfig.defaultSortOrder.property; this.sort.direction = (params.direction || this.entitiesTableConfig.defaultSortOrder.direction).toLowerCase() as SortDirection; if (params.hasOwnProperty('textSearch') && !isEmptyStr(params.textSearch)) { From fedd644516e58c1215640154e65ae6e904f385e6 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 9 Jun 2022 18:20:34 +0300 Subject: [PATCH 186/262] UI: move change password with profile to security --- ui-ngx/src/app/core/auth/auth.service.ts | 11 +- .../change-password-dialog.component.html | 64 --------- .../change-password-dialog.component.scss | 17 --- .../change-password-dialog.component.ts | 70 ---------- .../home/pages/profile/profile.component.html | 18 --- .../home/pages/profile/profile.component.scss | 6 - .../home/pages/profile/profile.component.ts | 55 +------- .../home/pages/profile/profile.module.ts | 4 +- .../pages/security/security.component.html | 115 +++++++++++++--- .../pages/security/security.component.scss | 55 ++++++-- .../home/pages/security/security.component.ts | 127 +++++++++++++++++- .../assets/locale/locale.constant-en_US.json | 21 ++- .../assets/locale/locale.constant-ru_RU.json | 1 - .../assets/locale/locale.constant-uk_UA.json | 1 - .../assets/locale/locale.constant-zh_CN.json | 1 - ui-ngx/src/theme.scss | 7 + 16 files changed, 305 insertions(+), 268 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.html delete mode 100644 ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.scss delete mode 100644 ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.ts diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 527c4cb6de..e26d7813da 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -23,7 +23,7 @@ import { catchError, map, mergeMap, tap } from 'rxjs/operators'; import { LoginRequest, LoginResponse, PublicLoginRequest } from '@shared/models/login.models'; import { ActivatedRoute, Router, UrlTree } from '@angular/router'; -import { defaultHttpOptions } from '../http/http-utils'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from '../http/http-utils'; import { UserService } from '../http/user.service'; import { Store } from '@ngrx/store'; import { AppState } from '../core.state'; @@ -47,6 +47,7 @@ import { AlertDialogComponent } from '@shared/components/dialog/alert-dialog.com import { OAuth2ClientInfo, PlatformType } from '@shared/models/oauth2.models'; import { isMobileApp } from '@core/utils'; import { TwoFactorAuthProviderType, TwoFaProviderInfo } from '@shared/models/two-factor-auth.models'; +import { UserPasswordPolicy } from '@shared/models/settings.models'; @Injectable({ providedIn: 'root' @@ -163,14 +164,18 @@ export class AuthService { )); } - public changePassword(currentPassword: string, newPassword: string) { - return this.http.post('/api/auth/changePassword', {currentPassword, newPassword}, defaultHttpOptions()).pipe( + public changePassword(currentPassword: string, newPassword: string, config?: RequestConfig) { + return this.http.post('/api/auth/changePassword', {currentPassword, newPassword}, defaultHttpOptionsFromConfig(config)).pipe( tap((loginResponse: LoginResponse) => { this.setUserFromJwtToken(loginResponse.token, loginResponse.refreshToken, false); } )); } + public getUserPasswordPolicy() { + return this.http.get(`/api/noauth/userPasswordPolicy`, defaultHttpOptions()); + } + public activateByEmailCode(emailCode: string): Observable { return this.http.post(`/api/noauth/activateByEmailCode?emailCode=${emailCode}`, null, defaultHttpOptions()); diff --git a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.html b/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.html deleted file mode 100644 index b5f6d81459..0000000000 --- a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.html +++ /dev/null @@ -1,64 +0,0 @@ - -
- -

profile.change-password

- - - - - -
-
- - profile.current-password - - lock - - - - login.new-password - - lock - - - - login.new-password-again - - lock - - -
-
- - -
-
diff --git a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.scss b/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.scss deleted file mode 100644 index 2d8406456a..0000000000 --- a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.scss +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host { -} diff --git a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.ts deleted file mode 100644 index 782bcb0f4d..0000000000 --- a/ui-ngx/src/app/modules/home/pages/profile/change-password-dialog.component.ts +++ /dev/null @@ -1,70 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, OnInit } from '@angular/core'; -import { MatDialogRef } from '@angular/material/dialog'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { FormBuilder, FormGroup } from '@angular/forms'; -import { ActionNotificationShow } from '@core/notification/notification.actions'; -import { TranslateService } from '@ngx-translate/core'; -import { AuthService } from '@core/auth/auth.service'; -import { DialogComponent } from '@shared/components/dialog.component'; -import { Router } from '@angular/router'; - -@Component({ - selector: 'tb-change-password-dialog', - templateUrl: './change-password-dialog.component.html', - styleUrls: ['./change-password-dialog.component.scss'] -}) -export class ChangePasswordDialogComponent extends DialogComponent implements OnInit { - - changePassword: FormGroup; - - constructor(protected store: Store, - protected router: Router, - private translate: TranslateService, - private authService: AuthService, - public dialogRef: MatDialogRef, - public fb: FormBuilder) { - super(store, router, dialogRef); - } - - ngOnInit(): void { - this.buildChangePasswordForm(); - } - - buildChangePasswordForm() { - this.changePassword = this.fb.group({ - currentPassword: [''], - newPassword: [''], - newPassword2: [''] - }); - } - - onChangePassword(): void { - if (this.changePassword.get('newPassword').value !== this.changePassword.get('newPassword2').value) { - this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('login.passwords-mismatch-error'), - type: 'error' })); - } else { - this.authService.changePassword( - this.changePassword.get('currentPassword').value, - this.changePassword.get('newPassword').value).subscribe(() => { - this.dialogRef.close(true); - }); - } - } -} diff --git a/ui-ngx/src/app/modules/home/pages/profile/profile.component.html b/ui-ngx/src/app/modules/home/pages/profile/profile.component.html index 7300f7ea7c..3a15616055 100644 --- a/ui-ngx/src/app/modules/home/pages/profile/profile.component.html +++ b/ui-ngx/src/app/modules/home/pages/profile/profile.component.html @@ -78,24 +78,6 @@ {{ 'dashboard.home-dashboard-hide-toolbar' | translate }} -
-
- -
-
- -
{{ expirationJwtData }}
-
-
-
{{ expirationJwtData }}
+ + +
+
+
+

profile.change-password

+ + profile.current-password + + + + {{ 'security.password-requirement.incorrect-password-try-again' | translate }} + + + + login.new-password + + + + {{ 'security.password-requirement.password-not-meet-requirements' | translate }} + + + {{ changePassword.get('newPassword').getError('alreadyUsed') }} + + + {{ 'security.password-requirement.password-should-difference' | translate }} + + + {{ 'security.password-requirement.password-should-not-contain-spaces' | translate }} + + +
+ +
+ + login.new-password-again + + + + {{ 'security.password-requirement.new-passwords-not-match' | translate }} + + +
+ +
+ +
+
+ +
+

security.password-requirement.password-requirements

+

security.password-requirement.at-least

+

+ + {{ 'security.password-requirement.uppercase-letter' | translate : {count: passwordPolicy.minimumUppercaseLetters} }} +

+

+ + {{ 'security.password-requirement.lowercase-letter' | translate : {count: passwordPolicy.minimumLowercaseLetters} }} +

+

+ + {{ 'security.password-requirement.digit' | translate : {count: passwordPolicy.minimumDigits} }} +

+

+ + {{ 'security.password-requirement.special-character' | translate : {count: passwordPolicy.minimumSpecialCharacters} }} +

+

+ + {{ 'security.password-requirement.character' | translate : {count: passwordPolicy.minimumLength} }} +

+
+ +
+ + +
+
+ + - admin.2fa.2fa + admin.2fa.2fa
security.2fa.2fa-description
diff --git a/ui-ngx/src/app/modules/home/pages/security/security.component.scss b/ui-ngx/src/app/modules/home/pages/security/security.component.scss index 7dad927038..16f19e09bf 100644 --- a/ui-ngx/src/app/modules/home/pages/security/security.component.scss +++ b/ui-ngx/src/app/modules/home/pages/security/security.component.scss @@ -21,6 +21,7 @@ } mat-card.profile-card { + padding: 24px; @media #{$mat-gt-sm} { width: 70%; } @@ -31,23 +32,54 @@ width: 45%; } - .mat-subheader { - line-height: 24px; - color: rgba(0, 0, 0, 0.54); + .card-title { + font: 500 18px / 24px Roboto, "Helvetica Neue", sans-serif; + letter-spacing: 0.15px; + margin-top: 0; + } + + .mat-h4 { + font-weight: 500; font-size: 14px; - font-weight: 400; + letter-spacing: 0.25px; + margin: 0 0 4px; + } + + .change-password { + margin: 0; + + .mat-divider.mat-divider-vertical { + margin-bottom: 25px; + } + + .mat-form-field { + margin-bottom: 4px; + } + + .password-requirements > p { + margin: 0 0 8px; + letter-spacing: 0.25px; + color: rgba(0, 0, 0, 0.87); + } + + .mat-icon[data-mat-icon-name="check"] { + color: #24A148; + } } - .profile-last-login-ts { - font-size: 16px; - font-weight: 400; + .auth-title { + font-weight: 500; + margin: 0; } - .profile-btn-subtext { + .token-text { font: 400 14px / 16px Roboto, "Helvetica Neue", sans-serif; letter-spacing: 0.25px; - opacity: 0.6; padding: 8px 0; + + > .date { + opacity: .7; + } } } @@ -91,3 +123,8 @@ } } } +:host ::ng-deep { + .mat-form-field-appearance-fill .mat-form-field-underline::before { + background-color: transparent; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/security/security.component.ts b/ui-ngx/src/app/modules/home/pages/security/security.component.ts index ab7d470e9e..8b723a0ab5 100644 --- a/ui-ngx/src/app/modules/home/pages/security/security.component.ts +++ b/ui-ngx/src/app/modules/home/pages/security/security.component.ts @@ -19,7 +19,15 @@ import { User } from '@shared/models/user.model'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormBuilder, FormGroup } from '@angular/forms'; +import { + AbstractControl, + FormBuilder, + FormGroup, FormGroupDirective, + NgForm, + ValidationErrors, + ValidatorFn, + Validators +} from '@angular/forms'; import { TranslateService } from '@ngx-translate/core'; import { MatDialog } from '@angular/material/dialog'; import { DialogService } from '@core/services/dialog.service'; @@ -39,7 +47,9 @@ import { import { authenticationDialogMap } from '@home/pages/security/authentication-dialog/authentication-dialog.map'; import { takeUntil, tap } from 'rxjs/operators'; import { Observable, of, Subject } from 'rxjs'; -import { isDefinedAndNotNull } from '@core/utils'; +import { isDefinedAndNotNull, isEqual } from '@core/utils'; +import { AuthService } from '@core/auth/auth.service'; +import { UserPasswordPolicy } from '@shared/models/settings.models'; @Component({ selector: 'tb-security', @@ -52,7 +62,11 @@ export class SecurityComponent extends PageComponent implements OnInit, OnDestro private accountConfig: AccountTwoFaSettings; twoFactorAuth: FormGroup; + changePassword: FormGroup; + user: User; + passwordPolicy: UserPasswordPolicy; + allowTwoFactorProviders: TwoFactorAuthProviderType[] = []; providersData = twoFactorAuthProvidersData; twoFactorAuthProviderType = TwoFactorAuthProviderType; @@ -80,6 +94,7 @@ export class SecurityComponent extends PageComponent implements OnInit, OnDestro public dialogService: DialogService, public fb: FormBuilder, private datePipe: DatePipe, + private authService: AuthService, private clipboardService: ClipboardService) { super(store); } @@ -88,6 +103,8 @@ export class SecurityComponent extends PageComponent implements OnInit, OnDestro this.buildTwoFactorForm(); this.user = this.route.snapshot.data.user; this.twoFactorLoad(this.route.snapshot.data.providers); + this.buildChangePasswordForm(); + this.loadPasswordPolicy(); } ngOnDestroy() { @@ -142,6 +159,75 @@ export class SecurityComponent extends PageComponent implements OnInit, OnDestro }); } + private buildChangePasswordForm() { + this.changePassword = this.fb.group({ + currentPassword: [''], + newPassword: ['', Validators.required], + newPassword2: ['', this.samePasswordValidation(false, 'newPassword')] + }); + } + + private loadPasswordPolicy() { + this.authService.getUserPasswordPolicy().subscribe(policy => { + this.passwordPolicy = policy; + this.changePassword.get('newPassword').setValidators([ + this.passwordStrengthValidator(), + this.samePasswordValidation(true, 'currentPassword'), + Validators.required + ]); + this.changePassword.get('newPassword').updateValueAndValidity({emitEvent: false}); + }); + } + + private passwordStrengthValidator(): ValidatorFn { + return (control: AbstractControl): ValidationErrors | null => { + const value: string = control.value; + const errors: any = {}; + + if (this.passwordPolicy.minimumUppercaseLetters > 0 && + !new RegExp(`(?:.*?[A-Z]){${this.passwordPolicy.minimumUppercaseLetters}}`).test(value)) { + errors.notUpperCase = true; + } + + if (this.passwordPolicy.minimumLowercaseLetters > 0 && + !new RegExp(`(?:.*?[a-z]){${this.passwordPolicy.minimumLowercaseLetters}}`).test(value)) { + errors.notLowerCase = true; + } + + if (this.passwordPolicy.minimumDigits > 0 + && !new RegExp(`(?:.*?\\d){${this.passwordPolicy.minimumDigits}}`).test(value)) { + errors.notNumeric = true; + } + + if (this.passwordPolicy.minimumSpecialCharacters > 0 && + !new RegExp(`(?:.*?[\\W_]){${this.passwordPolicy.minimumSpecialCharacters}}`).test(value)) { + errors.notSpecial = true; + } + + if (!this.passwordPolicy.allowWhitespaces && /\s/.test(value)) { + errors.hasWhitespaces = true; + } + + if (this.passwordPolicy.minimumLength > 0 && value.length < this.passwordPolicy.minimumLength) { + errors.minLength = true; + } + + return isEqual(errors, {}) ? null : errors; + }; + } + + private samePasswordValidation(isSame: boolean, key: string): ValidatorFn { + return (control: AbstractControl): ValidationErrors | null => { + const value: string = control.value; + const keyValue = control.parent?.value[key]; + + if (isSame) { + return value === keyValue ? {samePassword: true} : null; + } + return value !== keyValue ? {differencePassword: true} : null; + }; + } + trackByProvider(i: number, provider: TwoFactorAuthProviderType) { return provider; } @@ -256,4 +342,41 @@ export class SecurityComponent extends PageComponent implements OnInit, OnDestro } return info; } + + onChangePassword(form: FormGroupDirective): void { + if (this.changePassword.valid) { + this.authService.changePassword(this.changePassword.get('currentPassword').value, + this.changePassword.get('newPassword').value, {ignoreErrors: true}).subscribe(() => { + this.discardChanges(form); + }, + (error) => { + if (error.status === 400 && error.error.message === 'Current password doesn\'t match!') { + this.changePassword.get('currentPassword').setErrors({differencePassword: true}); + } else if (error.status === 400 && error.error.message.startsWith('Password must')) { + this.loadPasswordPolicy(); + } else if (error.status === 400 && error.error.message.startsWith('Password was already used')) { + this.changePassword.get('newPassword').setErrors({alreadyUsed: error.error.message}); + } else { + this.store.dispatch(new ActionNotificationShow({ + message: error.error.message, + type: 'error', + target: 'changePassword' + })); + } + }); + } else { + this.changePassword.markAllAsTouched(); + } + } + + discardChanges(form: FormGroupDirective, event?: MouseEvent) { + if (event) { + event.stopPropagation(); + } + form.resetForm({ + currentPassword: '', + newPassword: '', + newPassword2: '' + }); + } } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2ae0b55916..31d5229fa6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2498,7 +2498,7 @@ "password-reset": "Password reset", "expired-password-reset-message": "Your credentials has been expired! Please create new password.", "new-password": "New password", - "new-password-again": "New password again", + "new-password-again": "Confirm new password", "password-link-sent-message": "Reset link has been sent", "email": "Email", "login-with": "Login with {{name}}", @@ -2601,12 +2601,13 @@ "change-password": "Change Password", "current-password": "Current password", "copy-jwt-token": "Copy JWT token", - "valid-till": "Valid till {{expirationData}}", + "jwt-token": "JWT token", + "token-valid-till": "Token is valid till", "tokenCopiedSuccessMessage": "JWT token has been copied to clipboard", "tokenCopiedWarnMessage": "JWT token is expired! Please, refresh the page." }, "security": { - "security": "Security", + "security": "Password and authentication", "2fa": { "2fa": "Two-factor authentication", "2fa-description": "Two-factor authentication protects your account from unauthorized access. All you have to do is enter a security code when you log in.", @@ -2660,6 +2661,20 @@ "backup-code-description": "These printable one-time passcodes allow you to sign in when away from your phone, like when you’re traveling.", "backup-code-hint": "{{ info }} single-use codes are active at this time" } + }, + "password-requirement": { + "at-least": "At least:", + "character": "{ count, plural, 1 {1 character} other {# characters} }", + "digit": "{ count, plural, 1 {1 digit} other {# digits} }", + "incorrect-password-try-again": "Incorrect password. Try again", + "lowercase-letter": "{ count, plural, 1 {1 lowercase letter} other {# lowercase letters} }", + "new-passwords-not-match": "New password didn't match", + "password-should-not-contain-spaces": "Your password should not contain spaces", + "password-not-meet-requirements": "Password didn't meet requirements", + "password-requirements": "Password requirements", + "password-should-difference": "New password should be different from current", + "special-character": "{ count, plural, 1 {1 special character} other {# special characters} }", + "uppercase-letter": "{ count, plural, 1 {1 uppercase letter} other {# uppercase letters} }" } }, "relation": { diff --git a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json index a4e894e59e..b576974b9f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-ru_RU.json +++ b/ui-ngx/src/assets/locale/locale.constant-ru_RU.json @@ -1289,7 +1289,6 @@ "change-password": "Изменить пароль", "current-password": "Текущий пароль", "copy-jwt-token": "Копировать JWT токен", - "valid-till": "Действителен до {{expirationData}}", "tokenCopiedMessage": "JWT токен скопирован в буфер обмена", "tokenCopiedWarnMessage": "JWT токен недействителен! Перезагрузите страницу." }, diff --git a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json index eff83577fd..c624175bec 100644 --- a/ui-ngx/src/assets/locale/locale.constant-uk_UA.json +++ b/ui-ngx/src/assets/locale/locale.constant-uk_UA.json @@ -1704,7 +1704,6 @@ "change-password": "Змінити пароль", "current-password": "Поточний пароль", "copy-jwt-token": "Копіювати JWT токен", - "valid-till": "Дійсний до {{expirationData}}", "tokenCopiedMessage": "JWT токен скопійовано в буфер обміну", "tokenCopiedWarnMessage": "JWT токен не є дійсним! Перезавантажте сторінку." }, diff --git a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json index 11e34c9102..f4b6dd420f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json +++ b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json @@ -2009,7 +2009,6 @@ "last-login-time": "最后登录", "profile": "属性", "copy-jwt-token": "复制 JWT 令牌", - "valid-till": "有效期至 {{expirationData}}", "tokenCopiedSuccessMessage": "JWT 令牌已复制到剪贴板", "tokenCopiedWarnMessage": "JWT 令牌已过期！请刷新页面。" }, diff --git a/ui-ngx/src/theme.scss b/ui-ngx/src/theme.scss index e0aac57674..cdf4d611cd 100644 --- a/ui-ngx/src/theme.scss +++ b/ui-ngx/src/theme.scss @@ -221,6 +221,7 @@ $tb-dark-theme: get-tb-dark-theme( @mixin tb-components-theme($theme) { $primary: map-get($theme, primary); + $warn: map-get($theme, warn); mat-toolbar{ &.mat-hue-3 { @@ -233,6 +234,12 @@ $tb-dark-theme: get-tb-dark-theme( div.tb-dashboard-page.mobile-app { @include mat-fab-toolbar-inverse-theme($tb-theme); } + + .same-color.mat-form-field-invalid { + .mat-form-field-suffix { + color: mat.get-color-from-palette($warn, text); + } + } } .tb-default { From dbbd8cf9ce37320a1aeaf0bb4e4979f5573be173 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 10 Jun 2022 10:09:59 +0300 Subject: [PATCH 187/262] UI: Imporvement mobile view security page --- .../app/modules/home/pages/security/security.component.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/security/security.component.scss b/ui-ngx/src/app/modules/home/pages/security/security.component.scss index 16f19e09bf..b593be3a2a 100644 --- a/ui-ngx/src/app/modules/home/pages/security/security.component.scss +++ b/ui-ngx/src/app/modules/home/pages/security/security.component.scss @@ -23,10 +23,10 @@ mat-card.profile-card { padding: 24px; @media #{$mat-gt-sm} { - width: 70%; + width: 80%; } @media #{$mat-gt-md} { - width: 50%; + width: 55%; } @media #{$mat-gt-xl} { width: 45%; From eeb6ab6a6076c5f2481f00fa01d5897c781daba0 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 10 Jun 2022 10:34:31 +0300 Subject: [PATCH 188/262] UI: Refactoring code style --- .../login/two-factor-auth-login.component.ts | 15 +++++++-------- .../app/shared/models/two-factor-auth.models.ts | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.ts b/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.ts index 89105cd18e..2a98f0b530 100644 --- a/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.ts +++ b/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.ts @@ -29,7 +29,7 @@ import { import { TranslateService } from '@ngx-translate/core'; import { interval, Subscription } from 'rxjs'; import { isEqual } from '@core/utils'; -import {ActionNotificationShow} from "@core/notification/notification.actions"; +import { ActionNotificationShow } from '@core/notification/notification.actions'; @Component({ selector: 'tb-two-factor-auth-login', @@ -120,13 +120,12 @@ export class TwoFactorAuthLoginComponent extends PageComponent implements OnInit this.verificationForm.get('verificationCode').setErrors(errors); }, 5000); } else { - this.store.dispatch(new ActionNotificationShow( - { - message: error.error.message, - type: 'error', - verticalPosition: 'top', - horizontalPosition: 'left' - })); + this.store.dispatch(new ActionNotificationShow({ + message: error.error.message, + type: 'error', + verticalPosition: 'top', + horizontalPosition: 'left' + })); } } ); diff --git a/ui-ngx/src/app/shared/models/two-factor-auth.models.ts b/ui-ngx/src/app/shared/models/two-factor-auth.models.ts index d7961050ed..d3b2a74fbd 100644 --- a/ui-ngx/src/app/shared/models/two-factor-auth.models.ts +++ b/ui-ngx/src/app/shared/models/two-factor-auth.models.ts @@ -93,8 +93,8 @@ export interface AccountTwoFaSettings { } export type AccountTwoFaSettingProviders = { - [key in TwoFactorAuthProviderType]?: TwoFactorAuthAccountConfig; -} + [key in TwoFactorAuthProviderType]?: TwoFactorAuthAccountConfig; +}; export interface TwoFaProviderInfo { type: TwoFactorAuthProviderType; From fd53c24346f84d05b7114f918d7664afab4d0408 Mon Sep 17 00:00:00 2001 From: dlandiak Date: Fri, 10 Jun 2022 13:24:20 +0300 Subject: [PATCH 189/262] fixing slow queries logging --- .../server/dao/sql/query/DefaultEntityQueryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index f4fa052648..23b5b12d7c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -369,7 +369,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { try { return jdbcTemplate.queryForObject(countQuery, ctx, Long.class); } finally { - queryLog.logQuery(ctx, ctx.getQuery(), System.currentTimeMillis() - startTs); + queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs); } }); } @@ -481,7 +481,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { try { rows = jdbcTemplate.queryForList(dataQuery, ctx); } finally { - queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs); + queryLog.logQuery(ctx, dataQuery, System.currentTimeMillis() - startTs); } return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements); }); From 9ff4a9e72521d72e74c87086c0d8bcea44c36d9d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 10 Jun 2022 13:27:12 +0300 Subject: [PATCH 190/262] UI: Widgets bindles version control support --- .../pages/widget/widget-library.module.ts | 4 +- .../widget/widgets-bundle-tabs.component.html | 23 ++++++++++ .../widget/widgets-bundle-tabs.component.ts | 43 +++++++++++++++++++ .../widgets-bundles-table-config.resolver.ts | 2 + .../app/shared/models/entity-type.models.ts | 3 ++ ui-ngx/src/app/shared/models/vc.models.ts | 3 +- .../app/shared/models/widgets-bundle.model.ts | 4 +- .../assets/locale/locale.constant-en_US.json | 3 ++ 8 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.ts diff --git a/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts b/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts index f8a87d3960..fafb45d132 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts +++ b/ui-ngx/src/app/modules/home/pages/widget/widget-library.module.ts @@ -24,6 +24,7 @@ import { WidgetLibraryComponent } from './widget-library.component'; import { WidgetEditorComponent } from '@home/pages/widget/widget-editor.component'; import { SelectWidgetTypeDialogComponent } from '@home/pages/widget/select-widget-type-dialog.component'; import { SaveWidgetTypeAsDialogComponent } from './save-widget-type-as-dialog.component'; +import { WidgetsBundleTabsComponent } from '@home/pages/widget/widgets-bundle-tabs.component'; @NgModule({ declarations: [ @@ -31,7 +32,8 @@ import { SaveWidgetTypeAsDialogComponent } from './save-widget-type-as-dialog.co WidgetLibraryComponent, WidgetEditorComponent, SelectWidgetTypeDialogComponent, - SaveWidgetTypeAsDialogComponent + SaveWidgetTypeAsDialogComponent, + WidgetsBundleTabsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.html b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.html new file mode 100644 index 0000000000..0a90a829ee --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.html @@ -0,0 +1,23 @@ + + + + diff --git a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.ts b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.ts new file mode 100644 index 0000000000..00fef3adb8 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundle-tabs.component.ts @@ -0,0 +1,43 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityTabsComponent } from '../../components/entity/entity-tabs.component'; +import { WidgetsBundle } from '@shared/models/widgets-bundle.model'; +import { NULL_UUID } from '@shared/models/id/has-uuid'; + +@Component({ + selector: 'tb-widgets-bundle-tabs', + templateUrl: './widgets-bundle-tabs.component.html', + styleUrls: [] +}) +export class WidgetsBundleTabsComponent extends EntityTabsComponent { + + constructor(protected store: Store) { + super(store); + } + + isTenantWidgetsBundle() { + return this.entity && this.entity.tenantId.id !== NULL_UUID; + } + + ngOnInit() { + super.ngOnInit(); + } + +} diff --git a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundles-table-config.resolver.ts index 9533af5782..b6b1396c11 100644 --- a/ui-ngx/src/app/modules/home/pages/widget/widgets-bundles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/widget/widgets-bundles-table-config.resolver.ts @@ -39,6 +39,7 @@ import { DialogService } from '@core/services/dialog.service'; import { ImportExportService } from '@home/components/import-export/import-export.service'; import { Direction } from '@shared/models/page/sort-order'; import { map } from 'rxjs/operators'; +import { WidgetsBundleTabsComponent } from '@home/pages/widget/widgets-bundle-tabs.component'; @Injectable() export class WidgetsBundlesTableConfigResolver implements Resolve> { @@ -55,6 +56,7 @@ export class WidgetsBundlesTableConfigResolver implements Resolve = [ EntityType.DASHBOARD, EntityType.CUSTOMER, EntityType.DEVICE_PROFILE, - EntityType.RULE_CHAIN + EntityType.RULE_CHAIN, + EntityType.WIDGETS_BUNDLE ]; export interface VersionCreateConfig { diff --git a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts index 106438a591..235190ead2 100644 --- a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts +++ b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetsBundleId } from '@shared/models/id/widgets-bundle-id'; -export interface WidgetsBundle extends BaseData { +export interface WidgetsBundle extends BaseData, ExportableEntity { tenantId: TenantId; alias: string; title: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 284fcaa763..19cd88b1a5 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1843,6 +1843,9 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", + "type-widgets-bundle": "Widgets bundle", + "type-widgets-bundles": "Widgets bundles", + "list-of-widgets-bundles": "{ count, plural, 1 {One widgets bundle} other {List of # widget bundles} }", "search": "Search entities", "selected-entities": "{ count, plural, 1 {1 entity} other {# entities} } selected", "entity-name": "Entity name", From 9c3febf6641c9c613846e5f4edace20816ec7cfd Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 10 Jun 2022 13:49:51 +0300 Subject: [PATCH 191/262] UI: Fixed not correct work vertical resize in knob widget --- .../home/components/widget/lib/rpc/knob.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.html index 0e396d1b07..58b7d7880c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.html @@ -15,8 +15,8 @@ limitations under the License. --> -
-
+
+
{{ value }} From 7c2283e3cd727425c9d9ed3eea1c47b7e99f56fa Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 10 Jun 2022 15:10:36 +0300 Subject: [PATCH 192/262] Add indexes for external_id --- .../src/main/data/upgrade/3.3.4/schema_update.sql | 8 ++++++++ dao/src/main/resources/sql/schema-entities-idx.sql | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index babb630e9e..a5184c27e6 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -29,6 +29,14 @@ ALTER TABLE customer ALTER TABLE widgets_bundle ADD COLUMN IF NOT EXISTS external_id UUID; +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); + ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index b74d935c12..b47462f94b 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -51,3 +51,17 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); From db840bf4d88f28e3b8adaf18da49bd8d6caaaa94 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 10 Jun 2022 15:45:26 +0300 Subject: [PATCH 193/262] UI: Fixed incorrect show edit widget panel when active add widget panel in dashboard page --- .../components/dashboard-page/dashboard-page.component.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts index c360846941..f9ec50504f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts @@ -1127,6 +1127,13 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC editWidget($event: Event, layoutCtx: DashboardPageLayoutContext, widget: Widget) { $event.stopPropagation(); + + if (this.isAddingWidget) { + this.onAddWidgetClosed(); + this.isAddingWidgetClosed = true; + this.isEditingWidgetClosed = false; + } + if (this.editingWidgetOriginal === widget) { this.onEditWidgetClosed(); } else { From cc52452d75b47fa3b8f315955cf1e250ae9ba1cf Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 10 Jun 2022 16:17:37 +0300 Subject: [PATCH 194/262] Remove admin settings on tenant deletion --- .../sync/vc/TbAbstractVersionControlSettingsService.java | 2 +- .../server/dao/settings/AdminSettingsService.java | 4 +++- .../thingsboard/server/dao/settings/AdminSettingsDao.java | 2 ++ .../server/dao/settings/AdminSettingsServiceImpl.java | 8 +++++++- .../server/dao/sql/settings/AdminSettingsRepository.java | 2 ++ .../server/dao/sql/settings/JpaAdminSettingsDao.java | 6 ++++++ .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 5 +++++ 7 files changed, 26 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java index c9f5f5ee4a..3a674eece0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -72,7 +72,7 @@ public abstract class TbAbstractVersionControlSettingsService { boolean removeByTenantIdAndKey(UUID tenantId, String key); + void removeByTenantId(UUID tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 22a279639a..7d6f931ef1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -71,9 +71,15 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { } @Override - public boolean deleteAdminSettings(TenantId tenantId, String key) { + public boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); Validator.validateString(key, "Incorrect key " + key); return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); } + + @Override + public void deleteAdminSettingsByTenantId(TenantId tenantId) { + adminSettingsDao.removeByTenantId(tenantId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 0e68327fdf..e607517f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -29,6 +29,8 @@ public interface AdminSettingsRepository extends JpaRepository Date: Fri, 10 Jun 2022 16:48:12 +0300 Subject: [PATCH 195/262] VC request timing out --- .../DefaultGitVersionControlQueueService.java | 34 ++++++++++++++++--- .../sync/vc/data/PendingGitRequest.java | 4 +++ .../src/main/resources/thingsboard.yml | 1 + 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 759bc83acf..f5585fac39 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -34,9 +35,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -53,6 +54,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.VersionControlRespon import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; @@ -67,7 +69,13 @@ import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -81,16 +89,22 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final TbClusterService clusterService; private final DataDecodingEncodingService encodingService; private final DefaultEntitiesVersionControlService entitiesVersionControlService; + private final SchedulerComponent scheduler; private final Map> pendingRequestMap = new HashMap<>(); + @Value("${queue.vc.request-timeout:60000}") + private int requestTimeout; + public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, - @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService) { + @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService, + SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.clusterService = clusterService; this.encodingService = encodingService; this.entitiesVersionControlService = entitiesVersionControlService; + this.scheduler = scheduler; } @Override @@ -275,6 +289,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var requestBody = enrichFunction.apply(newRequestProto(request, settings)); log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); + request.setTimeoutTask(scheduler.schedule(() -> { + processTimeout(request.getRequestId()); + }, requestTimeout, TimeUnit.MILLISECONDS)); } else { throw new RuntimeException("Future is already done!"); } @@ -338,12 +355,13 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public void processResponse(VersionControlResponseMsg vcResponseMsg) { UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); - PendingGitRequest request = pendingRequestMap.get(requestId); + PendingGitRequest request = pendingRequestMap.remove(requestId); if (request == null) { log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); return; } else { log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + request.getTimeoutTask().cancel(true); } var future = request.getFuture(); if (!StringUtils.isEmpty(vcResponseMsg.getError())) { @@ -399,6 +417,14 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + private void processTimeout(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null) { + log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); + pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); + } + } + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java index fbec600ed0..b34806a6ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -17,9 +17,11 @@ package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; +import java.util.concurrent.ScheduledFuture; @Getter public class PendingGitRequest { @@ -28,6 +30,8 @@ public class PendingGitRequest { private final UUID requestId; private final TenantId tenantId; private final SettableFuture future; + @Setter + private ScheduledFuture timeoutTask; public PendingGitRequest(TenantId tenantId) { this.createdTime = System.currentTimeMillis(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ac6fe34905..94f5aa3dc1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1039,6 +1039,7 @@ queue: partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" From bb8e6a04439bbb0ac56ac99279f2b799e376d450 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 10 Jun 2022 17:32:12 +0300 Subject: [PATCH 196/262] Fixed bug with incorrect display of delayed status --- .../modules/home/components/edge/edge-downlink-table-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/edge/edge-downlink-table-config.ts b/ui-ngx/src/app/modules/home/components/edge/edge-downlink-table-config.ts index 2ddc6637b3..2e3b55f32d 100644 --- a/ui-ngx/src/app/modules/home/components/edge/edge-downlink-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/edge/edge-downlink-table-config.ts @@ -138,7 +138,7 @@ export class EdgeDownlinkTableConfig extends EntityTableConfig Date: Fri, 10 Jun 2022 17:38:18 +0300 Subject: [PATCH 197/262] UI: Fix bug with filter and with entity name --- .../widget/lib/alarms-table-widget.component.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 03379a54e0..2412f59f5f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -290,9 +290,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, debounceTime(150), distinctUntilChanged(), tap(() => { - if (this.displayPagination) { - this.paginator.pageIndex = 0; - } + this.resetPageIndex(); this.updateData(); }) ) @@ -556,6 +554,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.ctx.detectChanges(); } + private resetPageIndex(): void { + if (this.displayPagination) { + this.paginator.pageIndex = 0; + } + } + private editAlarmFilter($event: Event) { if ($event) { $event.stopPropagation(); @@ -600,6 +604,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.pageLink.statusList = result.statusList; this.pageLink.severityList = result.severityList; this.pageLink.typeList = result.typeList; + this.resetPageIndex(); this.updateData(); } }); @@ -620,9 +625,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, exitFilterMode() { this.textSearchMode = false; this.pageLink.textSearch = null; - if (this.displayPagination) { - this.paginator.pageIndex = 0; - } + this.resetPageIndex(); this.updateData(); this.ctx.hideTitlePanel = false; this.ctx.detectChanges(true); @@ -959,7 +962,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, } else if (alarmField.value === alarmFields.severity.value) { return this.translate.instant(alarmSeverityTranslations.get(value)); } else if (alarmField.value === alarmFields.status.value) { - return this.translate.instant(alarmStatusTranslations.get(value)); + return alarmStatusTranslations.get(value) ? this.translate.instant(alarmStatusTranslations.get(value)) : value; } else if (alarmField.value === alarmFields.originatorType.value) { return this.translate.instant(entityTypeTranslations.get(value).type); } else { From 0cd24bb16e26e22691f21e391f5634f4f27ac2bb Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Sun, 12 Jun 2022 20:28:12 +0300 Subject: [PATCH 198/262] refactoring: alarm - test start --- .../server/controller/AlarmController.java | 2 +- .../DefaultTbNotificationEntityService.java | 26 +--- .../entitiy/alarm/DefaultTbAlarmService.java | 10 +- .../service/entitiy/alarm/TbAlarmService.java | 3 +- .../controller/AbstractControllerTest.java | 2 +- .../controller/AbstractNotifyEntityTest.java | 132 ++++++++++++++++++ .../server/controller/AbstractWebTest.java | 11 +- .../controller/BaseAlarmControllerTest.java | 119 +++++++++++++++- .../profile/TbDeviceProfileNodeTest.java | 8 +- 9 files changed, 269 insertions(+), 44 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 939e9da8a0..2651277ccc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -147,7 +147,7 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - return tbAlarmService.delete(alarm, getCurrentUser()); + return tbAlarmService.delete(alarm, getCurrentUser().getCustomerId(), getCurrentUser()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 3361b75d23..707747ae91 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.gateway_device.GatewayNotificationsService import org.thingsboard.server.service.security.model.SecurityUser; import java.util.List; +import java.util.Locale; @Slf4j @Service @@ -335,28 +336,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS return null; } - private EdgeEventActionType edgeTypeByActionType(ActionType actionType) { - switch (actionType) { - case ADDED: - return EdgeEventActionType.ADDED; - case UPDATED: - return EdgeEventActionType.UPDATED; - case ALARM_ACK: - return EdgeEventActionType.ALARM_ACK; - case ALARM_CLEAR: - return EdgeEventActionType.ALARM_CLEAR; - case DELETED: - return EdgeEventActionType.DELETED; - case RELATION_ADD_OR_UPDATE: - return EdgeEventActionType.RELATION_ADD_OR_UPDATE; - case RELATION_DELETED: - return EdgeEventActionType.RELATION_DELETED; - case ASSIGNED_TO_EDGE: - return EdgeEventActionType.ASSIGNED_TO_EDGE; - case UNASSIGNED_FROM_EDGE: - return EdgeEventActionType.UNASSIGNED_FROM_EDGE; - default: - return null; - } + public static EdgeEventActionType edgeTypeByActionType(ActionType actionType) { + return EdgeEventActionType.valueOf(actionType.toString().toUpperCase(Locale.ENGLISH)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 4930f54a5d..305fda2b2b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -77,12 +78,13 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public Boolean delete(Alarm alarm, SecurityUser user) throws ThingsboardException { + public Boolean delete(Alarm alarm, CustomerId customerId, SecurityUser user) throws ThingsboardException { + TenantId tenantId = alarm.getTenantId(); try { - List relatedEdgeIds = findRelatedEdgeIds(user.getTenantId(), alarm.getOriginator()); - notificationEntityService.notifyDeleteAlarm(user.getTenantId(), alarm, alarm.getOriginator(), user.getCustomerId(), + List relatedEdgeIds = findRelatedEdgeIds(tenantId, alarm.getOriginator()); + notificationEntityService.notifyDeleteAlarm(tenantId, alarm, alarm.getOriginator(), customerId, relatedEdgeIds, user, JacksonUtil.OBJECT_MAPPER.writeValueAsString(alarm)); - return alarmService.deleteAlarm(user.getTenantId(), alarm.getId()).isSuccessful(); + return alarmService.deleteAlarm(tenantId, alarm.getId()).isSuccessful(); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 7a6d7f6a81..f17fcb3b70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy.alarm; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbAlarmService { @@ -27,5 +28,5 @@ public interface TbAlarmService { void clear(Alarm alarm, SecurityUser user) throws ThingsboardException; - Boolean delete(Alarm alarm, SecurityUser user) throws ThingsboardException; + Boolean delete(Alarm alarm, CustomerId customerId, SecurityUser user) throws ThingsboardException; } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 98e4c12759..ed8d8632da 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -45,7 +45,7 @@ import static org.assertj.core.api.Assertions.assertThat; @EnableWebSocket @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Slf4j -public abstract class AbstractControllerTest extends AbstractWebTest { +public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { public static final String WS_URL = "ws://localhost:"; diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java new file mode 100644 index 0000000000..a5827d6f6d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.mockito.Mockito; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.model.ModelConstants; + +import java.util.Locale; + +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.thingsboard.server.service.entitiy.DefaultTbNotificationEntityService.edgeTypeByActionType; + +public abstract class AbstractNotifyEntityTest extends AbstractWebTest { + + @SpyBean + protected TbClusterService tbClusterService; + + @SpyBean + protected AuditLogService auditLogService; + + protected void testNotifyEntityOk(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType) { + testSendNotificationMsgToEdgeServiceOk(entityId, tenantId, actionType); + testLogEntityActionOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + testPushMsgToRuleEngineOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + Mockito.reset(tbClusterService, auditLogService); + } + + protected void testNotifyEntityDeleteOk(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType) { + Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdgeService(Mockito.any(), + Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.any()); + testLogEntityActionOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + testPushMsgToRuleEngineOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + } + + private void testNotifyEntityError(EntityId entityId, HasName entity, TenantId tenantId, + UserId userId, String userName, ActionType actionType, Exception exp, + Object... additionalInfo) { + CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); + EntityId entity_NULL_UUID = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(entity.getClass().toString().substring(entity.getClass().toString().lastIndexOf(".") + 1).toUpperCase(Locale.ENGLISH)), + ModelConstants.NULL_UUID); + testNotificationMsgToEdgeServiceNever(entityId); + if (additionalInfo.length > 0) { + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), + Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), + Mockito.any(exp.getClass()), Mockito.eq(additionalInfo)); + } else { + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), + Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), + Mockito.any(exp.getClass()), Mockito.isNull()); + } + Mockito.verify(tbClusterService, never()).pushMsgToRuleEngine(Mockito.any(), Mockito.any(entityId.getClass()), + Mockito.any(), Mockito.any()); + Mockito.reset(tbClusterService, auditLogService); + } + + protected void testNotifyEntityNever(EntityId entityId, HasName entity) { + testNotificationMsgToEdgeServiceNever(entityId); + testLogEntityActionNever(entityId, entity); + testPushMsgToRuleEngineNever(entityId); + } + + protected void testNotificationMsgToEdgeServiceNever(EntityId entityId) { + Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdgeService(Mockito.any(), + Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.any()); + } + + protected void testLogEntityActionNever(EntityId entityId, HasName entity) { + Mockito.verify(auditLogService, never()).logEntityAction(Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(entity.getClass()), + Mockito.any(), Mockito.any()); + } + + protected void testPushMsgToRuleEngineNever(EntityId entityId) { + Mockito.verify(tbClusterService, never()).pushMsgToRuleEngine(Mockito.any(), + Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any()); + } + + private void testLogEntityActionOk(HasName entity, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType) { + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), + Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), + Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); + } + + private void testPushMsgToRuleEngineOk(HasName entity, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType) { + + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), + Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), + Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); + } + + private void testSendNotificationMsgToEdgeServiceOk(EntityId entityId, TenantId tenantId, ActionType actionType) { + Mockito.verify(tbClusterService, times(1)).sendNotificationMsgToEdgeService(Mockito.eq(tenantId), + Mockito.isNull(), Mockito.eq(entityId), Mockito.isNull(), Mockito.isNull(), + Mockito.eq(edgeTypeByActionType(actionType))); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index b52c3028d6..2e5012851a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -75,11 +75,6 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.data.queue.ProcessingStrategyType; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.SubmitStrategy; -import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -137,7 +132,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected TenantId tenantId; protected UserId tenantAdminUserId; + protected CustomerId tenantAdminCustomerId; protected CustomerId customerId; + protected UserId customerUserId; @SuppressWarnings("rawtypes") private HttpMessageConverter mappingJackson2HttpMessageConverter; @@ -202,6 +199,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { tenantAdmin = createUserAndLogin(tenantAdmin, TENANT_ADMIN_PASSWORD); tenantAdminUserId = tenantAdmin.getId(); + tenantAdminCustomerId = tenantAdmin.getCustomerId(); Customer customer = new Customer(); customer.setTitle("Customer"); @@ -215,7 +213,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { customerUser.setCustomerId(savedCustomer.getId()); customerUser.setEmail(CUSTOMER_USER_EMAIL); - createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); + customerUser = createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); + customerUserId = customerUser.getId(); logout(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java index ce9b6916fc..16a9f993c3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java @@ -15,17 +15,21 @@ */ package org.thingsboard.server.controller; +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.audit.ActionType; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +@Slf4j public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public static final String TEST_ALARM_TYPE = "Test"; @@ -56,36 +60,68 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testCreateAlarmViaCustomer() throws Exception { loginCustomerUser(); - createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); logout(); } @Test public void testCreateAlarmViaTenant() throws Exception { loginTenantAdmin(); - createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); logout(); } @Test public void testUpdateAlarmViaCustomer() throws Exception { loginCustomerUser(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + alarm.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm, Alarm.class); Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); + + testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.UPDATED); logout(); } @Test public void testUpdateAlarmViaTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + alarm.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm, Alarm.class); Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); + + testNotifyEntityOk(updatedAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED); logout(); } @@ -93,9 +129,15 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + alarm.setSeverity(AlarmSeverity.MAJOR); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm", alarm).andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -103,9 +145,15 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + loginDifferentCustomer(); alarm.setSeverity(AlarmSeverity.MAJOR); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm", alarm).andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -113,7 +161,13 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaCustomer() throws Exception { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); + + testNotifyEntityDeleteOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.DELETED); logout(); } @@ -121,7 +175,13 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaTenant() throws Exception { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); + + testNotifyEntityDeleteOk(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.DELETED); logout(); } @@ -129,8 +189,15 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); + logout(); } @@ -139,7 +206,13 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); loginDifferentCustomer(); + + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); + logout(); } @@ -147,10 +220,17 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaCustomer() throws Exception { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); + Alarm foundAlarm = doGet("/api/alarm/" + alarm.getId(), Alarm.class); Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); + + testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_CLEAR); logout(); } @@ -158,10 +238,16 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaTenant() throws Exception { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); Alarm foundAlarm = doGet("/api/alarm/" + alarm.getId(), Alarm.class); Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); + + testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR); logout(); } @@ -169,10 +255,17 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testAcknowledgeAlarmViaCustomer() throws Exception { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/ack").andExpect(status().isOk()); + Alarm foundAlarm = doGet("/api/alarm/" + alarm.getId(), Alarm.class); Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.ACTIVE_ACK, foundAlarm.getStatus()); + + testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ACK); logout(); } @@ -181,7 +274,12 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); loginDifferentCustomer(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -190,7 +288,12 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -199,7 +302,12 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { loginCustomerUser(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); loginDifferentCustomer(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/ack").andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -208,7 +316,12 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { loginTenantAdmin(); Alarm alarm = createAlarm(TEST_ALARM_TYPE); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/alarm/" + alarm.getId() + "/ack").andExpect(status().isForbidden()); + + testNotifyEntityNever(alarm.getId(), alarm); logout(); } @@ -221,10 +334,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { .severity(AlarmSeverity.CRITICAL) .type(type) .build(); - alarm = doPost("/api/alarm", alarm, Alarm.class); Assert.assertNotNull(alarm); - return alarm; } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 4be8372df9..9aa53b41f7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -40,12 +40,12 @@ import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.AlarmRule; +import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; +import org.thingsboard.server.common.data.device.profile.CustomTimeScheduleItem; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.DurationAlarmConditionSpec; import org.thingsboard.server.common.data.device.profile.RepeatingAlarmConditionSpec; -import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule; -import org.thingsboard.server.common.data.device.profile.CustomTimeScheduleItem; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -70,10 +70,10 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.ArrayList; import java.util.Optional; import java.util.TreeMap; import java.util.UUID; @@ -1157,7 +1157,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(listListenableFutureActiveSchedule); TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); - Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) + Mockito.when(ctx.newMsg(Mockito.any(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) .thenReturn(theMsg); ObjectNode data = mapper.createObjectNode(); From 6351366aed3c6a20daa4d3ad64e5dbc754e287a6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Jun 2022 10:57:28 +0200 Subject: [PATCH 199/262] fixed device profile node --- .../rule/engine/profile/TbDeviceProfileNodeTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 4be8372df9..a8c38ce33f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.device.profile.CustomTimeScheduleItem; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; @@ -1157,7 +1158,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(listListenableFutureActiveSchedule); TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); - Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) + Mockito.when(ctx.newMsg(Mockito.any(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyString())) .thenReturn(theMsg); ObjectNode data = mapper.createObjectNode(); From b5f052cbc5761e4138b186e45be0daa3b07dd117 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 13 Jun 2022 12:17:15 +0300 Subject: [PATCH 200/262] Fix typo --- .../home/components/profile/alarm/alarm-schedule.component.html | 2 +- ...ustom_schedule_format.md => alarm_custom_schedule_format.md} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ui-ngx/src/assets/help/en_US/device-profile/{alarm_сustom_schedule_format.md => alarm_custom_schedule_format.md} (100%) diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html index d859cd9201..65fe9c9636 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-schedule.component.html @@ -74,7 +74,7 @@
- +
device-profile.schedule-days
diff --git a/ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md b/ui-ngx/src/assets/help/en_US/device-profile/alarm_custom_schedule_format.md similarity index 100% rename from ui-ngx/src/assets/help/en_US/device-profile/alarm_сustom_schedule_format.md rename to ui-ngx/src/assets/help/en_US/device-profile/alarm_custom_schedule_format.md From 7983f0ee07e0d0f7ee86a8cf3093d1f6946e758e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Jun 2022 12:18:44 +0300 Subject: [PATCH 201/262] Performance improvements and refactoring --- .../DefaultEntitiesExportImportService.java | 7 +- .../sync/ie/EntitiesExportImportService.java | 3 +- .../DefaultExportableEntitiesService.java | 17 ---- .../exporting/ExportableEntitiesService.java | 5 -- .../impl/DefaultEntityExportService.java | 9 -- .../ie/importing/EntityImportService.java | 3 +- .../ie/importing/impl/AssetImportService.java | 3 +- .../impl/BaseEntityImportService.java | 90 ++++++++++--------- .../importing/impl/CustomerImportService.java | 7 +- .../impl/DashboardImportService.java | 12 +-- .../importing/impl/DeviceImportService.java | 5 +- .../impl/DeviceProfileImportService.java | 3 +- .../impl/RuleChainImportService.java | 15 ++-- .../impl/WidgetsBundleImportService.java | 11 +-- .../DefaultEntitiesVersionControlService.java | 27 +++--- .../sync/vc/data/EntitiesImportCtx.java | 75 ++++++++++++++++ 16 files changed, 174 insertions(+), 118 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 4e9e737076..b69fb5906c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.Collection; import java.util.Comparator; @@ -74,9 +75,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } @Override - public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, boolean saveReferences, boolean sendEvents) throws ThingsboardException { - if (!rateLimitService.checkEntityImportLimit(user.getTenantId())) { + if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { @@ -86,7 +87,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - EntityImportResult importResult = importService.importEntity(user, exportData, importSettings); + EntityImportResult importResult = importService.importEntity(ctx, exportData); if (saveReferences) { importResult.getSaveReferencesCallback().run(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 45e6fccc73..221b29622c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.Comparator; @@ -31,7 +32,7 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, boolean saveReferences, boolean sendEvents) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 9ea16477aa..fbc4dd4bc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -151,23 +151,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi entityRemover.accept(tenantId, id); } - - @Override - public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { - if (entity instanceof HasTenantId) { - accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); - } else { - accessControlService.checkPermission(user, Resource.of(entityType), operation); - } - } - - @Override - public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { - HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); - checkPermission(user, entity, entityId.getEntityType(), operation); - } - - @SuppressWarnings("unchecked") private Dao getDao(EntityType entityType) { return (Dao) daos.get(entityType); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 6ff2c96dcd..493ac0b9f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -40,9 +40,4 @@ public interface ExportableEntitiesService { void removeById(TenantId tenantId, I id); - - void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; - - void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException; - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index f2ed8cd243..1c1d70b0ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -66,7 +66,6 @@ public class DefaultEntityExportService relations = new ArrayList<>(); List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } relations.addAll(inboundRelations); List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } relations.addAll(outboundRelations); return relations; } private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { - exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES); - List scopes; if (entity.getId().getEntityType() == EntityType.DEVICE) { scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index bf249937b9..4e7d4d8495 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -23,10 +23,11 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; public interface EntityImportService, D extends EntityExportData> { - EntityImportResult importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException; + EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException; EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 164dad6bcd..dd34160c6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent @@ -43,7 +44,7 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider, EntityImportSettings importSettings) { + protected Asset prepareAndSave(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 3ba5bfcdbd..7ba59bb3c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -53,6 +53,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; @@ -81,54 +82,53 @@ public abstract class BaseEntityImportService importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException { + public EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { E entity = exportData.getEntity(); - E existingEntity = findExistingEntity(user.getTenantId(), entity, importSettings); + E existingEntity = findExistingEntity(ctx, entity); entity.setExternalId(entity.getId()); EntityImportResult importResult = new EntityImportResult<>(); - IdProvider idProvider = new IdProvider(user, importSettings, importResult); - setOwner(user.getTenantId(), entity, idProvider); + IdProvider idProvider = new IdProvider(ctx, importResult); + setOwner(ctx.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); - exportableEntitiesService.checkPermission(user, entity, getEntityType(), Operation.CREATE); } else { entity.setId(existingEntity.getId()); entity.setCreatedTime(existingEntity.getCreatedTime()); - exportableEntitiesService.checkPermission(user, existingEntity, getEntityType(), Operation.WRITE); } - E savedEntity = prepareAndSave(user.getTenantId(), entity, exportData, idProvider, importSettings); + E savedEntity = prepareAndSave(ctx, entity, exportData, idProvider); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); importResult.setEntityType(getEntityType()); - processAfterSaved(user, importResult, exportData, idProvider, importSettings); + processAfterSaved(ctx, importResult, exportData, idProvider); + + ctx.putInternalId(exportData.getExternalId(), savedEntity.getId()); return importResult; } protected abstract void setOwner(TenantId tenantId, E entity, IdProvider idProvider); - protected abstract E prepareAndSave(TenantId tenantId, E entity, D exportData, IdProvider idProvider, EntityImportSettings importSettings); + protected abstract E prepareAndSave(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider); - protected void processAfterSaved(SecurityUser user, EntityImportResult importResult, D exportData, - IdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { + protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult importResult, D exportData, IdProvider idProvider) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); importResult.addSendEventsCallback(() -> { - onEntitySaved(user, savedEntity, oldEntity); + onEntitySaved(ctx.getUser(), savedEntity, oldEntity); }); - if (importSettings.isUpdateRelations() && exportData.getRelations() != null) { - importRelations(user, exportData.getRelations(), importResult); + if (ctx.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(ctx.getUser(), exportData.getRelations(), importResult); } - if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) { - importAttributes(user, exportData.getAttributes(), importResult); + if (ctx.isSaveAttributes() && exportData.getAttributes() != null) { + importAttributes(ctx.getUser(), exportData.getAttributes(), importResult); } } @@ -138,12 +138,10 @@ public abstract class BaseEntityImportService to = findInternalEntity(user.getTenantId(), relation.getTo()); - exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); relation.setTo(to.getId()); } if (!relation.getFrom().equals(entity.getId())) { HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); - exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); } } @@ -155,15 +153,6 @@ public abstract class BaseEntityImportService { entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null, @@ -234,12 +223,12 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) + protected E findExistingEntity(EntitiesImportCtx ctx, E entity) { + return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) .or(() -> { - if (importSettings.isFindExistingByName()) { - return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); + if (ctx.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName())); } else { return Optional.empty(); } @@ -255,22 +244,27 @@ public abstract class BaseEntityImportService importResult; public ID getInternalId(ID externalId) { return getInternalId(externalId, true); - } + } public ID getInternalId(ID externalId, boolean throwExceptionIfNotFound) { if (externalId == null || externalId.isNullUid()) return null; + EntityId localId = ctx.getInternalId(externalId); + if (localId != null) { + return (ID) localId; + } + HasId entity; try { - entity = findInternalEntity(user.getTenantId(), externalId); + entity = findInternalEntity(ctx.getTenantId(), externalId); } catch (Exception e) { if (throwExceptionIfNotFound) { throw e; @@ -279,17 +273,26 @@ public abstract class BaseEntityImportService getInternalIdByUuid(UUID externalUuid) { if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty(); + for (EntityType entityType : EntityType.values()) { + EntityId externalId; + try { + externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); + } catch (Exception e) { + continue; + } + EntityId internalId = ctx.getInternalId(externalId); + if (internalId != null) { + return Optional.of(internalId); + } + } + for (EntityType entityType : EntityType.values()) { EntityId externalId; try { @@ -301,12 +304,13 @@ public abstract class BaseEntityImportService exportData, IdProvider idProvider, EntityImportSettings importSettings) { + protected Customer prepareAndSave(EntitiesImportCtx ctx, Customer customer, EntityExportData exportData, IdProvider idProvider) { if (!customer.isPublic()) { return customerService.saveCustomer(customer); } else { - Customer publicCustomer = customerService.findOrCreatePublicCustomer(tenantId); + Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId()); publicCustomer.setExternalId(customer.getExternalId()); - return customerDao.save(tenantId, publicCustomer); + return customerDao.save(ctx.getTenantId(), publicCustomer); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 9e681abd65..c720f6d0c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; import java.util.ArrayList; @@ -58,16 +59,17 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider, EntityImportSettings importSettings) { + protected Dashboard prepareAndSave(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); JsonNode configuration = dashboard.getConfiguration(); JsonNode entityAliases = configuration.get("entityAliases"); if (entityAliases != null && entityAliases.isObject()) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 9a3730bba9..0ad482e925 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent @@ -42,11 +43,11 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider, EntityImportSettings importSettings) { + protected DeviceProfile prepareAndSave(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 84d04959d9..f10377d005 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; @@ -52,16 +53,16 @@ public class RuleChainImportService extends BaseEntityImportService { @@ -85,8 +86,8 @@ public class RuleChainImportService extends BaseEntityImportService existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(tenantId, savedWidgetsBundle.getAlias()).stream() + Map existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(ctx.getTenantId(), savedWidgetsBundle.getAlias()).stream() .collect(Collectors.toMap(BaseWidgetType::getAlias, w -> w)); for (WidgetTypeDetails widget : exportData.getWidgets()) { WidgetTypeInfo existingWidget; @@ -70,13 +71,13 @@ public class WidgetsBundleImportService extends BaseEntityImportService widgetTypeService.deleteWidgetType(tenantId, widgetTypeId)); + .forEach(widgetTypeId -> widgetTypeService.deleteWidgetType(ctx.getTenantId(), widgetTypeId)); } return savedWidgetsBundle; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6737bca5e8..2cc4e16b1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesServic import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; @@ -244,13 +245,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) { try { - EntityImportResult importResult = exportImportService.importEntity(user, entityData, - EntityImportSettings.builder() + var ctx = new EntitiesImportCtx(user, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) .findExistingByName(false) - .build(), true, true); + .build()); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, true, true); return VersionLoadResult.success(EntityTypeLoadResult.builder() .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) @@ -269,6 +270,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List saveReferencesCallbacks = new ArrayList<>(); List sendEventsCallbacks = new ArrayList<>(); + EntitiesImportCtx ctx = new EntitiesImportCtx(user); + List entityTypes = request.getEntityTypes().keySet().stream() .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); for (EntityType entityType : entityTypes) { @@ -285,21 +288,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } - EntityImportSettings importSettings = EntityImportSettings.builder() + ctx.setSettings(EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .findExistingByName(config.isFindExistingEntityByName()) - .build(); + .build()); for (EntityExportData entityData : entityDataList) { EntityImportResult importResult; try { - importResult = exportImportService.importEntity(user, entityData, - importSettings, false, false); + importResult = exportImportService.importEntity(ctx, entityData, false, false); } catch (Exception e) { throw new LoadEntityException(entityData, e); } if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - toReimport.put(entityData.getEntity().getExternalId(), importSettings); + toReimport.put(entityData.getEntity().getExternalId(), ctx.getSettings()); continue; } @@ -324,8 +326,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get(); importSettings.setResetExternalIdsOfAnotherTenant(true); - EntityImportResult importResult = exportImportService.importEntity(user, entityData, - importSettings, false, false); + ctx.setSettings(importSettings); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, false, false); EntityTypeLoadResult stats = results.get(externalId.getEntityType()); if (importResult.getOldEntity() == null) stats.setCreated(stats.getCreated() + 1); @@ -347,11 +349,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); }, 100, entity -> { if (importedEntities.get(entityType) == null || !importedEntities.get(entityType).contains(entity.getId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } exportableEntitiesService.removeById(user.getTenantId(), entity.getId()); sendEventsCallbacks.add(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java new file mode 100644 index 0000000000..4fbdb28257 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class EntitiesImportCtx { + + private final SecurityUser user; + private EntityImportSettings settings; + + private final Map externalToInternalIdMap = new HashMap<>(); + + public EntitiesImportCtx(SecurityUser user) { + this(user, null); + } + + public EntitiesImportCtx(SecurityUser user, EntityImportSettings settings) { + this.user = user; + this.settings = settings; + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + public boolean isFindExistingByName() { + return getSettings().isFindExistingByName(); + } + + public boolean isUpdateRelations() { + return getSettings().isUpdateRelations(); + } + + public boolean isSaveAttributes() { + return getSettings().isSaveAttributes(); + } + + public boolean isSaveCredentials() { + return getSettings().isSaveCredentials(); + } + + public boolean isResetExternalIdsOfAnotherTenant() { + return getSettings().isResetExternalIdsOfAnotherTenant(); + } + + public EntityId getInternalId(EntityId externalId) { + return externalToInternalIdMap.get(externalId); + } + + public void putInternalId(EntityId externalId, EntityId internalId) { + externalToInternalIdMap.put(externalId, internalId); + } +} From e78a891044cba9623c3a52fe3ed81d19c787c2b3 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 13 Jun 2022 13:47:45 +0300 Subject: [PATCH 202/262] Entity views export/import --- .../main/data/upgrade/3.3.4/schema_update.sql | 2 + .../DefaultEntitiesExportImportService.java | 2 +- .../impl/EntityViewImportService.java | 47 +++++++++++++++++++ .../server/common/data/EntityView.java | 5 +- .../server/common/data/sync/JsonTbEntity.java | 2 + .../server/dao/entityview/EntityViewDao.java | 3 +- .../model/sql/AbstractEntityViewEntity.java | 10 ++++ .../sql/entityview/EntityViewRepository.java | 3 +- .../dao/sql/entityview/JpaEntityViewDao.java | 16 ++++++- .../main/resources/sql/schema-entities.sql | 3 +- .../app/shared/models/entity-view.models.ts | 4 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + 12 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index babb630e9e..124e3eb9bf 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -28,6 +28,8 @@ ALTER TABLE customer ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE widgets_bundle ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE entity_view + ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 4e9e737076..052e9b9279 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -57,7 +57,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE, - EntityType.WIDGETS_BUNDLE + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE ); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java new file mode 100644 index 0000000000..a7156a290b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class EntityViewImportService extends BaseEntityImportService> { + + private final EntityViewService entityViewService; + + @Override + protected void setOwner(TenantId tenantId, EntityView entityView, IdProvider idProvider) { + entityView.setTenantId(tenantId); + entityView.setCustomerId(idProvider.getInternalId(entityView.getCustomerId())); + } + + @Override + protected EntityView prepareAndSave(TenantId tenantId, EntityView entityView, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { + entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); + return entityViewService.saveEntityView(entityView); + } + + @Override + protected void onEntitySaved(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { + entityNotificationService.notifyCreateOrUpdateEntity(user.getTenantId(), savedEntityView.getId(), savedEntityView, + null, oldEntityView == null ? ActionType.ADDED : ActionType.UPDATED, user); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java index 1f238fd672..a7e21b8a25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @AllArgsConstructor @EqualsAndHashCode(callSuper = true) public class EntityView extends SearchTextBasedWithAdditionalInfo - implements HasName, HasTenantId, HasCustomerId { + implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 5582010124562018986L; @@ -59,6 +59,8 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo @ApiModelProperty(position = 10, value = "Represents the end time of the interval that is used to limit access to target device telemetry. Customer will not be able to see entity telemetry that is outside the specified interval;") private long endTimeMs; + private EntityViewId externalId; + public EntityView() { super(); } @@ -77,6 +79,7 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo this.keys = entityView.getKeys(); this.startTimeMs = entityView.getStartTimeMs(); this.endTimeMs = entityView.getEndTimeMs(); + this.externalId = entityView.getExternalId(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index 4bb636c73b..0e2929bef8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.widget.WidgetsBundle; @@ -43,6 +44,7 @@ import java.lang.annotation.Target; @Type(name = "ASSET", value = Asset.class), @Type(name = "DASHBOARD", value = Dashboard.class), @Type(name = "CUSTOMER", value = Customer.class), + @Type(name = "ENTITY_VIEW", value = EntityView.class), @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundle.class) }) public @interface JsonTbEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index f44bafe937..49d1b94ecc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.List; import java.util.Optional; @@ -31,7 +32,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/28/2017. */ -public interface EntityViewDao extends Dao { +public interface EntityViewDao extends Dao, ExportableEntityDao { /** * Find entity view info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index a8a31cd892..cc68adb2a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -89,6 +89,9 @@ public abstract class AbstractEntityViewEntity extends Bas @Column(name = ModelConstants.ENTITY_VIEW_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + private static final ObjectMapper mapper = new ObjectMapper(); public AbstractEntityViewEntity() { @@ -121,6 +124,9 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityView.getEndTimeMs(); this.searchText = entityView.getSearchText(); this.additionalInfo = entityView.getAdditionalInfo(); + if (entityView.getExternalId() != null) { + this.externalId = entityView.getExternalId().getId(); + } } public AbstractEntityViewEntity(EntityViewEntity entityViewEntity) { @@ -137,6 +143,7 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityViewEntity.getEndTs(); this.searchText = entityViewEntity.getSearchText(); this.additionalInfo = entityViewEntity.getAdditionalInfo(); + this.externalId = entityViewEntity.getExternalId(); } @Override @@ -172,6 +179,9 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setStartTimeMs(startTs); entityView.setEndTimeMs(endTs); entityView.setAdditionalInfo(additionalInfo); + if (externalId != null) { + entityView.setExternalId(new EntityViewId(externalId)); + } return entityView; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 7a88aa3c5b..b8e7c1b90d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/31/2017. */ -public interface EntityViewRepository extends JpaRepository { +public interface EntityViewRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.EntityViewInfoEntity(e, c.title, c.additionalInfo) " + "FROM EntityViewEntity e " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index f77db1d8cc..133eb2be2a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -201,9 +201,23 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findEntityViewsByTenantId(tenantId, pageLink); + } + + @Override + public EntityView findByTenantIdAndName(UUID tenantId, String name) { + return findEntityViewByTenantIdAndName(tenantId, name).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.ENTITY_VIEW; } - } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index e232e5deb2..09b5a2947c 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -440,7 +440,8 @@ CREATE TABLE IF NOT EXISTS entity_view ( start_ts bigint, end_ts bigint, search_text varchar(255), - additional_info varchar + additional_info varchar, + external_id uuid ); CREATE TABLE IF NOT EXISTS ts_kv_latest diff --git a/ui-ngx/src/app/shared/models/entity-view.models.ts b/ui-ngx/src/app/shared/models/entity-view.models.ts index 42b76de788..9605edfc85 100644 --- a/ui-ngx/src/app/shared/models/entity-view.models.ts +++ b/ui-ngx/src/app/shared/models/entity-view.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntityViewId } from '@shared/models/id/entity-view-id'; @@ -32,7 +32,7 @@ export interface TelemetryEntityView { attributes: AttributesEntityView; } -export interface EntityView extends BaseData { +export interface EntityView extends BaseData, ExportableEntity { tenantId: TenantId; customerId: CustomerId; entityId: EntityId; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index f22bc0ad67..b0485cc37e 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -28,6 +28,7 @@ export const exportableEntityTypes: Array = [ EntityType.CUSTOMER, EntityType.DEVICE_PROFILE, EntityType.RULE_CHAIN, + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE ]; From 67137e563dd0a52e003a8d963a1e737f5660e0d2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 13 Jun 2022 14:38:34 +0300 Subject: [PATCH 203/262] License header for EntityViewImportService --- .../importing/impl/EntityViewImportService.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java index a7156a290b..9bbb57fb4f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; From 82763495e00a0a3cfe2e79069e68104c229a2d88 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Jun 2022 16:21:05 +0300 Subject: [PATCH 204/262] Perofrmance improvements --- .../DefaultEntitiesExportImportService.java | 1 + .../impl/BaseEntityImportService.java | 38 +++++++++---------- .../impl/DashboardImportService.java | 5 +-- .../impl/RuleChainImportService.java | 5 +-- .../DefaultEntitiesVersionControlService.java | 28 ++++++++++---- .../sync/vc/data/EntitiesImportCtx.java | 7 +++- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../thingsboard/common/util/TbStopWatch.java | 13 ++++++- .../rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 9 files changed, 64 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index b69fb5906c..5ef7b84ce7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -96,6 +96,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS importResult.getSendEventsCallback().run(); } + ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); return importResult; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ba59bb3c4..f1cef3191c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -45,12 +45,10 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @@ -83,13 +81,14 @@ public abstract class BaseEntityImportService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { + EntityImportResult importResult = new EntityImportResult<>(); + IdProvider idProvider = new IdProvider(ctx, importResult); + E entity = exportData.getEntity(); - E existingEntity = findExistingEntity(ctx, entity); + E existingEntity = findExistingEntity(ctx, entity, idProvider); entity.setExternalId(entity.getId()); - EntityImportResult importResult = new EntityImportResult<>(); - IdProvider idProvider = new IdProvider(ctx, importResult); setOwner(ctx.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); @@ -125,39 +124,38 @@ public abstract class BaseEntityImportService relations, EntityImportResult importResult) { + private void importRelations(EntitiesImportCtx ctx, List relations, EntityImportResult importResult, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); E entity = importResult.getSavedEntity(); importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { if (!relation.getTo().equals(entity.getId())) { - HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(to.getId()); + relation.setTo(idProvider.getInternalId(relation.getTo())); } if (!relation.getFrom().equals(entity.getId())) { - HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(from.getId()); + relation.setFrom(idProvider.getInternalId(relation.getFrom())); } } if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { - relationService.deleteRelation(user.getTenantId(), existingRelation); + relationService.deleteRelation(tenantId, existingRelation); importResult.addSendEventsCallback(() -> { - entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null, + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, ActionType.RELATION_DELETED, null, existingRelation); - entityActionService.logEntityAction(user, existingRelation.getTo(), null, null, + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, ActionType.RELATION_DELETED, null, existingRelation); }); } @@ -165,11 +163,11 @@ public abstract class BaseEntityImportService { - entityActionService.logEntityAction(user, relation.getFrom(), null, null, + entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, ActionType.RELATION_ADD_OR_UPDATE, null, relation); - entityActionService.logEntityAction(user, relation.getTo(), null, null, + entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, ActionType.RELATION_ADD_OR_UPDATE, null, relation); }); } @@ -223,7 +221,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) .or(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index c720f6d0c2..24238c78e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -59,8 +58,8 @@ public class DashboardImportService extends BaseEntityImportService importResult = exportImportService.importEntity(ctx, entityData, true, true); return VersionLoadResult.success(EntityTypeLoadResult.builder() .entityType(importResult.getEntityType()) @@ -271,6 +272,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List sendEventsCallbacks = new ArrayList<>(); EntitiesImportCtx ctx = new EntitiesImportCtx(user); + var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); @@ -294,6 +296,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .findExistingByName(config.isFindExistingEntityByName()) .build()); for (EntityExportData entityData : entityDataList) { + sw.startNew("Entities " + entityType.name()); + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { importResult = exportImportService.importEntity(ctx, entityData, false, false); @@ -322,6 +326,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + sw.startNew("Reimport"); toReimport.forEach((externalId, importSettings) -> { try { EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get(); @@ -341,6 +346,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } }); + sw.startNew("Remove Others"); request.getEntityTypes().keySet().stream() .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) @@ -361,6 +367,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); }); + sw.startNew("Callbacks"); + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { saveReferencesCallback.run(); @@ -375,6 +383,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont log.error("Failed to send events for entity", e); } } + + sw.stop(); + for (var task : sw.getTaskInfo()) { + log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); + } + log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); return VersionLoadResult.success(new ArrayList<>(results.values())); } @@ -412,8 +426,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportCredentials(otherVersion.hasCredentials()) .build()); return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion.sort()), - JacksonUtil.toPrettyString(otherVersion.sort())), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); }, MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 4fbdb28257..93de254d2a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; @@ -24,6 +25,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.HashMap; import java.util.Map; +@Slf4j @Data public class EntitiesImportCtx { @@ -66,10 +68,13 @@ public class EntitiesImportCtx { } public EntityId getInternalId(EntityId externalId) { - return externalToInternalIdMap.get(externalId); + var result = externalToInternalIdMap.get(externalId); + log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); + return result; } public void putInternalId(EntityId externalId, EntityId internalId) { + log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId); externalToInternalIdMap.put(externalId, internalId); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e38677b2ce..9fbe9b1104 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate { log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); From eb105d0859c5a1e2510f6e39a4d268fb8c73d391 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Mon, 13 Jun 2022 16:37:40 +0300 Subject: [PATCH 205/262] UI: Refactoring phone input --- .../components/phone-input.component.html | 4 +- .../components/phone-input.component.scss | 5 - .../components/phone-input.component.ts | 64 +- .../src/app/shared/models/country.models.ts | 1573 +++-------------- 4 files changed, 268 insertions(+), 1378 deletions(-) diff --git a/ui-ngx/src/app/shared/components/phone-input.component.html b/ui-ngx/src/app/shared/components/phone-input.component.html index d52d7202d0..4b81391b5f 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.html +++ b/ui-ngx/src/app/shared/components/phone-input.component.html @@ -22,8 +22,8 @@ {{ flagIcon }} - {{country.flag}} - {{country.name + ' +' + country.dialCode }} + {{country.flag}} + {{' ' + country.name + ' +' + country.dialCode }}
diff --git a/ui-ngx/src/app/shared/components/phone-input.component.scss b/ui-ngx/src/app/shared/components/phone-input.component.scss index 872c686af0..d8bd9df461 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.scss +++ b/ui-ngx/src/app/shared/components/phone-input.component.scss @@ -44,11 +44,6 @@ width: 45px; height: 30px; - .country-flag { - font-size: 22px; - padding-right: 10px; - } - .mat-select-trigger { height: 100%; width: 100%; diff --git a/ui-ngx/src/app/shared/components/phone-input.component.ts b/ui-ngx/src/app/shared/components/phone-input.component.ts index bbee8098f6..f5fdf62f34 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.ts +++ b/ui-ngx/src/app/shared/components/phone-input.component.ts @@ -62,15 +62,15 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida @Input() floatLabel: FloatLabelType = 'always'; @Input() appearance: MatFormFieldAppearance = 'legacy'; - allCountries: Array = []; + allCountries: Array = this.countryCodeData.allCountries; phonePlaceholder: string; - countryCallingCode: string; flagIcon: string; - + phoneFormGroup: FormGroup; phoneNumberPattern = phoneNumberPattern; + private baseCode = 127397; + private countryCallingCode: string; private modelValue: string; - public phoneFormGroup: FormGroup; private valueChange$: Subscription = null; private propagateChange = (v: any) => { }; @@ -80,17 +80,24 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida } ngOnInit(): void { - if (this.enableFlagsSelect) { - this.fetchCountryData(); + const validators: ValidatorFn[] = [Validators.pattern(phoneNumberPattern), this.validatePhoneNumber()]; + if (this.required) { + validators.push(Validators.required); } this.phoneFormGroup = this.fb.group({ country: [this.defaultCountry, []], - phoneNumber: [null, this.required ? [Validators.required] : []] + phoneNumber: [null, validators] }); - this.phoneFormGroup.get('phoneNumber').setValidators([Validators.pattern(phoneNumberPattern), this.validatePhoneNumber()]); - this.valueChange$ = this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(() => { + this.valueChange$ = this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(value => { this.updateModel(); + if (value) { + const parsedPhoneNumber = parsePhoneNumberFromString(value); + const country = this.phoneFormGroup.get('country').value; + if (parsedPhoneNumber?.country && parsedPhoneNumber?.country !== country) { + this.phoneFormGroup.get('country').patchValue(parsedPhoneNumber.country, {emitEvent: true}); + } + } }); this.phoneFormGroup.get('country').valueChanges.subscribe(value => { @@ -101,20 +108,8 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida if (phoneNumber) { if (code !== this.countryCallingCode && phoneNumber.includes(code)) { phoneNumber = phoneNumber.replace(code, this.countryCallingCode); - } else { - phoneNumber = this.countryCallingCode; + this.phoneFormGroup.get('phoneNumber').patchValue(phoneNumber); } - this.phoneFormGroup.get('phoneNumber').patchValue(phoneNumber); - } - } - }); - - this.phoneFormGroup.get('phoneNumber').valueChanges.subscribe(value => { - if (value) { - const parsedPhoneNumber = parsePhoneNumberFromString(value); - const country = this.phoneFormGroup.get('country').value; - if (parsedPhoneNumber?.country && parsedPhoneNumber?.country !== country) { - this.phoneFormGroup.get('country').patchValue(parsedPhoneNumber.country, {emitEvent: true}); } } }); @@ -135,22 +130,21 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida } } - getFlagAndPhoneNumberData(country) { + private getFlagAndPhoneNumberData(country) { if (this.enableFlagsSelect) { this.flagIcon = this.getFlagIcon(country); } this.getPhoneNumberData(country); } - getPhoneNumberData(country): void { + private getPhoneNumberData(country): void { const phoneData = getExampleNumber(country, examples); this.phonePlaceholder = phoneData.number; this.countryCallingCode = '+' + phoneData.countryCallingCode; } - getFlagIcon(countryCode) { - const base = 127462 - 65; - return String.fromCodePoint(...countryCode.split('').map(country => base + country.charCodeAt(0))); + private getFlagIcon(countryCode) { + return String.fromCodePoint(...countryCode.split('').map(country => this.baseCode + country.charCodeAt(0))); } validatePhoneNumber(): ValidatorFn { @@ -192,21 +186,6 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida } } - protected fetchCountryData(): void { - this.allCountries = []; - this.countryCodeData.allCountries.forEach((c) => { - const cc = c[1]; - const country: Country = { - name: c[0].toString(), - iso2: c[1].toString(), - dialCode: c[2].toString(), - flag: this.getFlagIcon(cc), - }; - - this.allCountries.push(country); - }); - } - writeValue(phoneNumber): void { this.modelValue = phoneNumber; const country = phoneNumber ? parsePhoneNumberFromString(phoneNumber)?.country : this.defaultCountry; @@ -223,5 +202,4 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida this.propagateChange(null); } } - } diff --git a/ui-ngx/src/app/shared/models/country.models.ts b/ui-ngx/src/app/shared/models/country.models.ts index 10612de015..d4895fada7 100644 --- a/ui-ngx/src/app/shared/models/country.models.ts +++ b/ui-ngx/src/app/shared/models/country.models.ts @@ -24,15 +24,7 @@ export interface Country { flag: string; } -export enum SearchCountryField { - DialCode = 'dialCode', - Iso2 = 'iso2', - Name = 'name', - All = 'all' -} - - -export enum CountryISO{ +export enum CountryISO { Afghanistan = 'AF', Albania = 'AL', Algeria = 'DZ', @@ -280,1324 +272,249 @@ export enum CountryISO{ @Injectable() export class CountryData { - public allCountries = [ - [ - 'Afghanistan', - CountryISO.Afghanistan, - '93' - ], - [ - 'Albania', - CountryISO.Albania, - '355' - ], - [ - 'Algeria', - CountryISO.Algeria, - '213' - ], - [ - 'American Samoa', - CountryISO.AmericanSamoa, - '1', - 1, - [ - '684', - ] - ], - [ - 'Andorra', - CountryISO.Andorra, - '376' - ], - [ - 'Angola', - CountryISO.Angola, - '244' - ], - [ - 'Anguilla', - CountryISO.Anguilla, - '1', - 1, - [ - '264', - ] - ], - [ - 'Antigua and Barbuda', - CountryISO.AntiguaAndBarbuda, - '1', - 1, - [ - '268', - ] - ], - [ - 'Argentina', - CountryISO.Argentina, - '54' - ], - [ - 'Armenia', - CountryISO.Armenia, - '374' - ], - [ - 'Aruba', - CountryISO.Aruba, - '297' - ], - [ - 'Australia', - CountryISO.Australia, - '61', - 0 - ], - [ - 'Austria', - CountryISO.Austria, - '43' - ], - [ - 'Azerbaijan', - CountryISO.Azerbaijan, - '994' - ], - [ - 'Bahamas', - CountryISO.Bahamas, - '1', - 1, - [ - '242', - ] - ], - [ - 'Bahrain', - CountryISO.Bahrain, - '973' - ], - [ - 'Bangladesh', - CountryISO.Bangladesh, - '880' - ], - [ - 'Barbados', - CountryISO.Barbados, - '1', - 1, - [ - '246', - ] - ], - [ - 'Belarus', - CountryISO.Belarus, - '375' - ], - [ - 'Belgium', - CountryISO.Belgium, - '32' - ], - [ - 'Belize', - CountryISO.Belize, - '501' - ], - [ - 'Benin', - CountryISO.Benin, - '229' - ], - [ - 'Bermuda', - CountryISO.Bermuda, - '1', - 1, - [ - '441', - ] - ], - [ - 'Bhutan', - CountryISO.Bhutan, - '975' - ], - [ - 'Bolivia', - CountryISO.Bolivia, - '591' - ], - [ - 'Bosnia and Herzegovina', - CountryISO.BosniaAndHerzegovina, - '387' - ], - [ - 'Botswana', - CountryISO.Botswana, - '267' - ], - [ - 'Brazil', - CountryISO.Brazil, - '55' - ], - [ - 'British Indian Ocean Territory', - CountryISO.BritishIndianOceanTerritory, - '246' - ], - [ - 'British Virgin Islands', - CountryISO.BritishVirginIslands, - '1', - 1, - [ - '284', - ] - ], - [ - 'Brunei', - CountryISO.Brunei, - '673' - ], - [ - 'Bulgaria', - CountryISO.Bulgaria, - '359' - ], - [ - 'Burkina Faso', - CountryISO.BurkinaFaso, - '226' - ], - [ - 'Burundi', - CountryISO.Burundi, - '257' - ], - [ - 'Cambodia', - CountryISO.Cambodia, - '855' - ], - [ - 'Cameroon', - CountryISO.Cameroon, - '237' - ], - [ - 'Canada', - CountryISO.Canada, - '1', - 1, - [ - '204', '226', '236', '249', '250', '289', '306', '343', '365', '387', '403', '416', - '418', '431', '437', '438', '450', '506', '514', '519', '548', '579', '581', '587', - '604', '613', '639', '647', '672', '705', '709', '742', '778', '780', '782', '807', - '819', '825', '867', '873', '902', '905' - ] - ], - [ - 'Cape Verde', - CountryISO.CapeVerde, - '238' - ], - [ - 'Caribbean Netherlands', - CountryISO.CaribbeanNetherlands, - '599', - 1 - ], - [ - 'Cayman Islands', - CountryISO.CaymanIslands, - '1', - 1, - [ - '345', - ] - ], - [ - 'Central African Republic', - CountryISO.CentralAfricanRepublic, - '236' - ], - [ - 'Chad', - CountryISO.Chad, - '235' - ], - [ - 'Chile', - CountryISO.Chile, - '56' - ], - [ - 'China', - CountryISO.China, - '86' - ], - [ - 'Christmas Island', - CountryISO.ChristmasIsland, - '61', - 2 - ], - [ - 'Cocos Islands', - CountryISO.Cocos, - '61', - 1 - ], - [ - 'Colombia', - CountryISO.Colombia, - '57' - ], - [ - 'Comoros', - CountryISO.Comoros, - '269' - ], - [ - 'Congo-Kinshasa', - CountryISO.CongoDRCJamhuriYaKidemokrasiaYaKongo, - '243' - ], - [ - 'Congo-Brazzaville', - CountryISO.CongoRepublicCongoBrazzaville, - '242' - ], - [ - 'Cook Islands', - CountryISO.CookIslands, - '682' - ], - [ - 'Costa Rica', - CountryISO.CostaRica, - '506' - ], - [ - 'Côte d’Ivoire', - CountryISO.CôteDIvoire, - '225' - ], - [ - 'Croatia', - CountryISO.Croatia, - '385' - ], - [ - 'Cuba', - CountryISO.Cuba, - '53' - ], - [ - 'Curaçao', - CountryISO.Curaçao, - '599', - 0 - ], - [ - 'Cyprus', - CountryISO.Cyprus, - '357' - ], - [ - 'Czech Republic', - CountryISO.CzechRepublic, - '420' - ], - [ - 'Denmark', - CountryISO.Denmark, - '45' - ], - [ - 'Djibouti', - CountryISO.Djibouti, - '253' - ], - [ - 'Dominica', - CountryISO.Dominica, - '1767' - ], - [ - 'Dominican Republic', - CountryISO.DominicanRepublic, - '1', - 2, - ['809', '829', '849'] - ], - [ - 'Ecuador', - CountryISO.Ecuador, - '593' - ], - [ - 'Egypt', - CountryISO.Egypt, - '20' - ], - [ - 'El Salvador', - CountryISO.ElSalvador, - '503' - ], - [ - 'Equatorial Guinea', - CountryISO.EquatorialGuinea, - '240' - ], - [ - 'Eritrea', - CountryISO.Eritrea, - '291' - ], - [ - 'Estonia', - CountryISO.Estonia, - '372' - ], - [ - 'Ethiopia', - CountryISO.Ethiopia, - '251' - ], - [ - 'Falkland Islands', - CountryISO.FalklandIslands, - '500' - ], - [ - 'Faroe Islands', - CountryISO.FaroeIslands, - '298' - ], - [ - 'Fiji', - CountryISO.Fiji, - '679' - ], - [ - 'Finland', - CountryISO.Finland, - '358', - 0 - ], - [ - 'France', - CountryISO.France, - '33' - ], - [ - 'French Guiana', - CountryISO.FrenchGuiana, - '594' - ], - [ - 'French Polynesia', - CountryISO.FrenchPolynesia, - '689' - ], - [ - 'Gabon', - CountryISO.Gabon, - '241' - ], - [ - 'Gambia', - CountryISO.Gambia, - '220' - ], - [ - 'Georgia', - CountryISO.Georgia, - '995' - ], - [ - 'Germany', - CountryISO.Germany, - '49' - ], - [ - 'Ghana', - CountryISO.Ghana, - '233' - ], - [ - 'Gibraltar', - CountryISO.Gibraltar, - '350' - ], - [ - 'Greece', - CountryISO.Greece, - '30' - ], - [ - 'Greenland', - CountryISO.Greenland, - '299' - ], - [ - 'Grenada', - CountryISO.Grenada, - '1473' - ], - [ - 'Guadeloupe', - CountryISO.Guadeloupe, - '590', - 0 - ], - [ - 'Guam', - CountryISO.Guam, - '1', - 1, - [ - '671', - ] - ], - [ - 'Guatemala', - CountryISO.Guatemala, - '502' - ], - [ - 'Guernsey', - CountryISO.Guernsey, - '44', - 1, - [1481] - ], - [ - 'Guinea', - CountryISO.Guinea, - '224' - ], - [ - 'Guinea-Bissau', - CountryISO.GuineaBissau, - '245' - ], - [ - 'Guyana', - CountryISO.Guyana, - '592' - ], - [ - 'Haiti', - CountryISO.Haiti, - '509' - ], - [ - 'Honduras', - CountryISO.Honduras, - '504' - ], - [ - 'Hong Kong', - CountryISO.HongKong, - '852' - ], - [ - 'Hungary', - CountryISO.Hungary, - '36' - ], - [ - 'Iceland', - CountryISO.Iceland, - '354' - ], - [ - 'India', - CountryISO.India, - '91' - ], - [ - 'Indonesia', - CountryISO.Indonesia, - '62' - ], - [ - 'Iran', - CountryISO.Iran, - '98' - ], - [ - 'Iraq', - CountryISO.Iraq, - '964' - ], - [ - 'Ireland', - CountryISO.Ireland, - '353' - ], - [ - 'Isle of Man', - CountryISO.IsleOfMan, - '44', - 2, - [1624] - ], - [ - 'Israel', - CountryISO.Israel, - '972' - ], - [ - 'Italy', - CountryISO.Italy, - '39', - 0 - ], - [ - 'Jamaica', - CountryISO.Jamaica, - '1', - 1, - [ - '876', - ] - ], - [ - 'Japan', - CountryISO.Japan, - '81' - ], - [ - 'Jersey', - CountryISO.Jersey, - '44', - 3, - [1534] - ], - [ - 'Jordan', - CountryISO.Jordan, - '962' - ], - [ - 'Kazakhstan', - CountryISO.Kazakhstan, - '7', - 1 - ], - [ - 'Kenya', - CountryISO.Kenya, - '254' - ], - [ - 'Kiribati', - CountryISO.Kiribati, - '686' - ], - [ - 'Kosovo', - CountryISO.Kosovo, - '383' - ], - [ - 'Kuwait', - CountryISO.Kuwait, - '965' - ], - [ - 'Kyrgyzstan', - CountryISO.Kyrgyzstan, - '996' - ], - [ - 'Laos', - CountryISO.Laos, - '856' - ], - [ - 'Latvia', - CountryISO.Latvia, - '371' - ], - [ - 'Lebanon', - CountryISO.Lebanon, - '961' - ], - [ - 'Lesotho', - CountryISO.Lesotho, - '266' - ], - [ - 'Liberia', - CountryISO.Liberia, - '231' - ], - [ - 'Libya', - CountryISO.Libya, - '218' - ], - [ - 'Liechtenstein', - CountryISO.Liechtenstein, - '423' - ], - [ - 'Lithuania', - CountryISO.Lithuania, - '370' - ], - [ - 'Luxembourg', - CountryISO.Luxembourg, - '352' - ], - [ - 'Macau', - CountryISO.Macau, - '853' - ], - [ - 'Macedonia', - CountryISO.Macedonia, - '389' - ], - [ - 'Madagascar', - CountryISO.Madagascar, - '261' - ], - [ - 'Malawi', - CountryISO.Malawi, - '265' - ], - [ - 'Malaysia', - CountryISO.Malaysia, - '60' - ], - [ - 'Maldives', - CountryISO.Maldives, - '960' - ], - [ - 'Mali', - CountryISO.Mali, - '223' - ], - [ - 'Malta', - CountryISO.Malta, - '356' - ], - [ - 'Marshall Islands', - CountryISO.MarshallIslands, - '692' - ], - [ - 'Martinique', - CountryISO.Martinique, - '596' - ], - [ - 'Mauritania', - CountryISO.Mauritania, - '222' - ], - [ - 'Mauritius', - CountryISO.Mauritius, - '230' - ], - [ - 'Mayotte', - CountryISO.Mayotte, - '262', - 1 - ], - [ - 'Mexico', - CountryISO.Mexico, - '52' - ], - [ - 'Micronesia', - CountryISO.Micronesia, - '691' - ], - [ - 'Moldova', - CountryISO.Moldova, - '373' - ], - [ - 'Monaco', - CountryISO.Monaco, - '377' - ], - [ - 'Mongolia', - CountryISO.Mongolia, - '976' - ], - [ - 'Montenegro', - CountryISO.Montenegro, - '382' - ], - [ - 'Montserrat', - CountryISO.Montserrat, - '1', - 1, - [ - '664', - ] - ], - [ - 'Morocco', - CountryISO.Morocco, - '212', - 0 - ], - [ - 'Mozambique', - CountryISO.Mozambique, - '258' - ], - [ - 'Myanmar', - CountryISO.Myanmar, - '95' - ], - [ - 'Namibia', - CountryISO.Namibia, - '264' - ], - [ - 'Nauru', - CountryISO.Nauru, - '674' - ], - [ - 'Nepal', - CountryISO.Nepal, - '977' - ], - [ - 'Netherlands', - CountryISO.Netherlands, - '31' - ], - [ - 'New Caledonia', - CountryISO.NewCaledonia, - '687' - ], - [ - 'New Zealand', - CountryISO.NewZealand, - '64' - ], - [ - 'Nicaragua', - CountryISO.Nicaragua, - '505' - ], - [ - 'Niger', - CountryISO.Niger, - '227' - ], - [ - 'Nigeria', - CountryISO.Nigeria, - '234' - ], - [ - 'Niue', - CountryISO.Niue, - '683' - ], - [ - 'Norfolk Island', - CountryISO.NorfolkIsland, - '672' - ], - [ - 'North Korea', - CountryISO.NorthKorea, - '850' - ], - [ - 'Northern Mariana Islands', - CountryISO.NorthernMarianaIslands, - '1670' - ], - [ - 'Norway', - CountryISO.Norway, - '47', - 0 - ], - [ - 'Oman', - CountryISO.Oman, - '968' - ], - [ - 'Pakistan', - CountryISO.Pakistan, - '92' - ], - [ - 'Palau', - CountryISO.Palau, - '680' - ], - [ - 'Palestine', - CountryISO.Palestine, - '970' - ], - [ - 'Panama', - CountryISO.Panama, - '507' - ], - [ - 'Papua New Guinea', - CountryISO.PapuaNewGuinea, - '675' - ], - [ - 'Paraguay', - CountryISO.Paraguay, - '595' - ], - [ - 'Peru', - CountryISO.Peru, - '51' - ], - [ - 'Philippines', - CountryISO.Philippines, - '63' - ], - [ - 'Poland', - CountryISO.Poland, - '48' - ], - [ - 'Portugal', - CountryISO.Portugal, - '351' - ], - [ - 'Puerto Rico', - CountryISO.PuertoRico, - '1', - 3, - ['787', '939'] - ], - [ - 'Qatar', - CountryISO.Qatar, - '974' - ], - [ - 'Réunion', - CountryISO.Réunion, - '262', - 0 - ], - [ - 'Romania', - CountryISO.Romania, - '40' - ], - [ - 'Russia', - CountryISO.Russia, - '7', - 0 - ], - [ - 'Rwanda', - CountryISO.Rwanda, - '250' - ], - [ - 'Saint Barthélemy', - CountryISO.SaintBarthélemy, - '590', - 1 - ], - [ - 'Saint Helena', - CountryISO.SaintHelena, - '290' - ], - [ - 'Saint Kitts and Nevis', - CountryISO.SaintKittsAndNevis, - '1869' - ], - [ - 'Saint Lucia', - CountryISO.SaintLucia, - '1', - 1, - [ - '758', - ] - ], - [ - 'Saint Martin', - CountryISO.SaintMartin, - '590', - 2 - ], - [ - 'Saint Pierre and Miquelon', - CountryISO.SaintPierreAndMiquelon, - '508' - ], - [ - 'Saint Vincent and the Grenadines', - CountryISO.SaintVincentAndTheGrenadines, - '1', - 1, - [ - '784', - ] - ], - [ - 'Samoa', - CountryISO.Samoa, - '685' - ], - [ - 'San Marino', - CountryISO.SanMarino, - '378' - ], - [ - 'São Tomé and Príncipe', - CountryISO.SãoToméAndPríncipe, - '239' - ], - [ - 'Saudi Arabia', - CountryISO.SaudiArabia, - '966' - ], - [ - 'Senegal', - CountryISO.Senegal, - '221' - ], - [ - 'Serbia', - CountryISO.Serbia, - '381' - ], - [ - 'Seychelles', - CountryISO.Seychelles, - '248' - ], - [ - 'Sierra Leone', - CountryISO.SierraLeone, - '232' - ], - [ - 'Singapore', - CountryISO.Singapore, - '65' - ], - [ - 'Sint Maarten', - CountryISO.SintMaarten, - '1', - 1, - [ - '721', - ] - ], - [ - 'Slovakia', - CountryISO.Slovakia, - '421' - ], - [ - 'Slovenia', - CountryISO.Slovenia, - '386' - ], - [ - 'Solomon Islands', - CountryISO.SolomonIslands, - '677' - ], - [ - 'Somalia', - CountryISO.Somalia, - '252' - ], - [ - 'South Africa', - CountryISO.SouthAfrica, - '27' - ], - [ - 'South Korea', - CountryISO.SouthKorea, - '82' - ], - [ - 'South Sudan', - CountryISO.SouthSudan, - '211' - ], - [ - 'Spain', - CountryISO.Spain, - '34' - ], - [ - 'Sri Lanka', - CountryISO.SriLanka, - '94' - ], - [ - 'Sudan', - CountryISO.Sudan, - '249' - ], - [ - 'Suriname', - CountryISO.Suriname, - '597' - ], - [ - 'Svalbard and Jan Mayen', - CountryISO.SvalbardAndJanMayen, - '47', - 1 - ], - [ - 'Swaziland', - CountryISO.Swaziland, - '268' - ], - [ - 'Sweden', - CountryISO.Sweden, - '46' - ], - [ - 'Switzerland', - CountryISO.Switzerland, - '41' - ], - [ - 'Syria', - CountryISO.Syria, - '963' - ], - [ - 'Taiwan', - CountryISO.Taiwan, - '886' - ], - [ - 'Tajikistan', - CountryISO.Tajikistan, - '992' - ], - [ - 'Tanzania', - CountryISO.Tanzania, - '255' - ], - [ - 'Thailand', - CountryISO.Thailand, - '66' - ], - [ - 'Timor-Leste', - CountryISO.TimorLeste, - '670' - ], - [ - 'Togo', - CountryISO.Togo, - '228' - ], - [ - 'Tokelau', - CountryISO.Tokelau, - '690' - ], - [ - 'Tonga', - CountryISO.Tonga, - '676' - ], - [ - 'Trinidad and Tobago', - CountryISO.TrinidadAndTobago, - '1', - 1, - [ - '868', - ] - ], - [ - 'Tunisia', - CountryISO.Tunisia, - '216' - ], - [ - 'Turkey', - CountryISO.Turkey, - '90' - ], - [ - 'Turkmenistan', - CountryISO.Turkmenistan, - '993' - ], - [ - 'Turks and Caicos Islands', - CountryISO.TurksAndCaicosIslands, - '1649' - ], - [ - 'Tuvalu', - CountryISO.Tuvalu, - '688' - ], - [ - 'U.S. Virgin Islands', - CountryISO.USVirginIslands, - '1', - 1, - [ - '340', - ] - ], - [ - 'Uganda', - CountryISO.Uganda, - '256' - ], - [ - 'Ukraine', - CountryISO.Ukraine, - '380' - ], - [ - 'United Arab Emirates', - CountryISO.UnitedArabEmirates, - '971' - ], - [ - 'United Kingdom', - CountryISO.UnitedKingdom, - '44', - 0 - ], - [ - 'United States', - CountryISO.UnitedStates, - '1', - 0 - ], - [ - 'Uruguay', - CountryISO.Uruguay, - '598' - ], - [ - 'Uzbekistan', - CountryISO.Uzbekistan, - '998' - ], - [ - 'Vanuatu', - CountryISO.Vanuatu, - '678' - ], - [ - 'Vatican City', - CountryISO.VaticanCity, - '39', - 1 - ], - [ - 'Venezuela', - CountryISO.Venezuela, - '58' - ], - [ - 'Vietnam', - CountryISO.Vietnam, - '84' - ], - [ - 'Wallis and Futuna', - CountryISO.WallisAndFutuna, - '681' - ], - [ - 'Western Sahara', - CountryISO.WesternSahara, - '212', - 1 - ], - [ - 'Yemen', - CountryISO.Yemen, - '967' - ], - [ - 'Zambia', - CountryISO.Zambia, - '260' - ], - [ - 'Zimbabwe', - CountryISO.Zimbabwe, - '263' - ], - [ - 'Åland Islands', - CountryISO.ÅlandIslands, - '358', - 1 - ] + public allCountries: Array = [ + {name: 'Afghanistan', iso2: CountryISO.Afghanistan, dialCode: '93', flag: '🇦🇫'}, + {name: 'Albania', iso2: CountryISO.Albania, dialCode: '355', flag: '🇦🇱'}, + {name: 'Algeria', iso2: CountryISO.Algeria, dialCode: '213', flag: '🇩🇿'}, + {name: 'American Samoa', iso2: CountryISO.AmericanSamoa, dialCode: '1', flag: '🇦🇸'}, + {name: 'Andorra', iso2: CountryISO.Andorra, dialCode: '376', flag: '🇦🇩'}, + {name: 'Angola', iso2: CountryISO.Angola, dialCode: '244', flag: '🇦🇴'}, + {name: 'Anguilla', iso2: CountryISO.Anguilla, dialCode: '1', flag: '🇦🇮'}, + {name: 'Antigua and Barbuda', iso2: CountryISO.AntiguaAndBarbuda, dialCode: '1', flag: '🇦🇬'}, + {name: 'Argentina', iso2: CountryISO.Argentina, dialCode: '54', flag: '🇦🇷'}, + {name: 'Armenia', iso2: CountryISO.Armenia, dialCode: '374', flag: '🇦🇲'}, + {name: 'Aruba', iso2: CountryISO.Aruba, dialCode: '297', flag: '🇦🇼'}, + {name: 'Australia', iso2: CountryISO.Australia, dialCode: '61', flag: '🇦🇺'}, + {name: 'Austria', iso2: CountryISO.Austria, dialCode: '43', flag: '🇦🇹'}, + {name: 'Azerbaijan', iso2: CountryISO.Azerbaijan, dialCode: '994', flag: '🇦🇿'}, + {name: 'Bahamas', iso2: CountryISO.Bahamas, dialCode: '1', flag: '🇧🇸'}, + {name: 'Bahrain', iso2: CountryISO.Bahrain, dialCode: '973', flag: '🇧🇭'}, + {name: 'Bangladesh', iso2: CountryISO.Bangladesh, dialCode: '880', flag: '🇧🇩'}, + {name: 'Barbados', iso2: CountryISO.Barbados, dialCode: '1', flag: '🇧🇧'}, + {name: 'Belarus', iso2: CountryISO.Belarus, dialCode: '375', flag: '🇧🇾'}, + {name: 'Belgium', iso2: CountryISO.Belgium, dialCode: '32', flag: '🇧🇪'}, + {name: 'Belize', iso2: CountryISO.Belize, dialCode: '501', flag: '🇧🇿'}, + {name: 'Benin', iso2: CountryISO.Benin, dialCode: '229', flag: '🇧🇯'}, + {name: 'Bermuda', iso2: CountryISO.Bermuda, dialCode: '1', flag: '🇧🇲'}, + {name: 'Bhutan', iso2: CountryISO.Bhutan, dialCode: '975', flag: '🇧🇹'}, + {name: 'Bolivia', iso2: CountryISO.Bolivia, dialCode: '591', flag: '🇧🇴'}, + {name: 'Bosnia and Herzegovina', iso2: CountryISO.BosniaAndHerzegovina, dialCode: '387', flag: '🇧🇦'}, + {name: 'Botswana', iso2: CountryISO.Botswana, dialCode: '267', flag: '🇧🇼'}, + {name: 'Brazil', iso2: CountryISO.Brazil, dialCode: '55', flag: '🇧🇷'}, + {name: 'British Indian Ocean Territory', iso2: CountryISO.BritishIndianOceanTerritory, dialCode: '246', flag: '🇮🇴'}, + {name: 'British Virgin Islands', iso2: CountryISO.BritishVirginIslands, dialCode: '1', flag: '🇻🇬'}, + {name: 'Brunei', iso2: CountryISO.Brunei, dialCode: '673', flag: '🇧🇳'}, + {name: 'Bulgaria', iso2: CountryISO.Bulgaria, dialCode: '359', flag: '🇧🇬'}, + {name: 'Burkina Faso', iso2: CountryISO.BurkinaFaso, dialCode: '226', flag: '🇧🇫'}, + {name: 'Burundi', iso2: CountryISO.Burundi, dialCode: '257', flag: '🇧🇮'}, + {name: 'Cambodia', iso2: CountryISO.Cambodia, dialCode: '855', flag: '🇰🇭'}, + {name: 'Cameroon', iso2: CountryISO.Cameroon, dialCode: '237', flag: '🇨🇲'}, + {name: 'Canada', iso2: CountryISO.Canada, dialCode: '1', flag: '🇨🇦'}, + {name: 'Cape Verde', iso2: CountryISO.CapeVerde, dialCode: '238', flag: '🇨🇻'}, + {name: 'Caribbean Netherlands', iso2: CountryISO.CaribbeanNetherlands, dialCode: '599', flag: '🇧🇶'}, + {name: 'Cayman Islands', iso2: CountryISO.CaymanIslands, dialCode: '1', flag: '🇰🇾'}, + {name: 'Central African Republic', iso2: CountryISO.CentralAfricanRepublic, dialCode: '236', flag: '🇨🇫'}, + {name: 'Chad', iso2: CountryISO.Chad, dialCode: '235', flag: '🇹🇩'}, + {name: 'Chile', iso2: CountryISO.Chile, dialCode: '56', flag: '🇨🇱'}, + {name: 'China', iso2: CountryISO.China, dialCode: '86', flag: '🇨🇳'}, + {name: 'Christmas Island', iso2: CountryISO.ChristmasIsland, dialCode: '61', flag: '🇨🇽'}, + {name: 'Cocos Islands', iso2: CountryISO.Cocos, dialCode: '61', flag: '🇨🇨'}, + {name: 'Colombia', iso2: CountryISO.Colombia, dialCode: '57', flag: '🇨🇨'}, + {name: 'Comoros', iso2: CountryISO.Comoros, dialCode: '269', flag: '🇰🇲'}, + {name: 'Congo-Kinshasa', iso2: CountryISO.CongoDRCJamhuriYaKidemokrasiaYaKongo, dialCode: '243', flag: '🇨🇩'}, + {name: 'Congo-Brazzaville', iso2: CountryISO.CongoRepublicCongoBrazzaville, dialCode: '242', flag: '🇨🇬'}, + {name: 'Cook Islands', iso2: CountryISO.CookIslands, dialCode: '682', flag: '🇨🇰'}, + {name: 'Costa Rica', iso2: CountryISO.CostaRica, dialCode: '506', flag: '🇨🇷'}, + {name: 'Côte d’Ivoire', iso2: CountryISO.CôteDIvoire, dialCode: '225', flag: '🇨🇮'}, + {name: 'Croatia', iso2: CountryISO.Croatia, dialCode: '385', flag: '🇭🇷'}, + {name: 'Cuba', iso2: CountryISO.Cuba, dialCode: '53', flag: '🇨🇺'}, + {name: 'Curaçao', iso2: CountryISO.Curaçao, dialCode: '599', flag: '🇨🇼'}, + {name: 'Cyprus', iso2: CountryISO.Cyprus, dialCode: '357', flag: '🇨🇾'}, + {name: 'Czech Republic', iso2: CountryISO.CzechRepublic, dialCode: '420', flag: '🇨🇿'}, + {name: 'Denmark', iso2: CountryISO.Denmark, dialCode: '45', flag: '🇩🇰'}, + {name: 'Djibouti', iso2: CountryISO.Djibouti, dialCode: '253', flag: '🇩🇯'}, + {name: 'Dominica', iso2: CountryISO.Dominica, dialCode: '1767', flag: '🇩🇲'}, + {name: 'Dominican Republic', iso2: CountryISO.DominicanRepublic, dialCode: '1', flag: '🇩🇴'}, + {name: 'Ecuador', iso2: CountryISO.Ecuador, dialCode: '593', flag: '🇪🇨'}, + {name: 'Egypt', iso2: CountryISO.Egypt, dialCode: '20', flag: '🇪🇬'}, + {name: 'El Salvador', iso2: CountryISO.ElSalvador, dialCode: '503', flag: '🇸🇻'}, + {name: 'Equatorial Guinea', iso2: CountryISO.EquatorialGuinea, dialCode: '240', flag: '🇬🇶'}, + {name: 'Eritrea', iso2: CountryISO.Eritrea, dialCode: '291', flag: '🇪🇷'}, + {name: 'Estonia', iso2: CountryISO.Estonia, dialCode: '372', flag: '🇪🇪'}, + {name: 'Ethiopia', iso2: CountryISO.Ethiopia, dialCode: '251', flag: '🇪🇹'}, + {name: 'Falkland Islands', iso2: CountryISO.FalklandIslands, dialCode: '500', flag: '🇫🇰'}, + {name: 'Faroe Islands', iso2: CountryISO.FaroeIslands, dialCode: '298', flag: '🇫🇴'}, + {name: 'Fiji', iso2: CountryISO.Fiji, dialCode: '679', flag: '🇫🇯'}, + {name: 'Finland', iso2: CountryISO.Finland, dialCode: '358', flag: '🇫🇮'}, + {name: 'France', iso2: CountryISO.France, dialCode: '33', flag: '🇫🇷'}, + {name: 'French Guiana', iso2: CountryISO.FrenchGuiana, dialCode: '594', flag: '🇬🇫'}, + {name: 'French Polynesia', iso2: CountryISO.FrenchPolynesia, dialCode: '689', flag: '🇵🇫'}, + {name: 'Gabon', iso2: CountryISO.Gabon, dialCode: '241', flag: '🇬🇦'}, + {name: 'Gambia', iso2: CountryISO.Gambia, dialCode: '220', flag: '🇬🇲'}, + {name: 'Georgia', iso2: CountryISO.Georgia, dialCode: '995', flag: '🇬🇪'}, + {name: 'Germany', iso2: CountryISO.Germany, dialCode: '49', flag: '🇩🇪'}, + {name: 'Ghana', iso2: CountryISO.Ghana, dialCode: '233', flag: '🇬🇭'}, + {name: 'Gibraltar', iso2: CountryISO.Gibraltar, dialCode: '350', flag: '🇬🇮'}, + {name: 'Greece', iso2: CountryISO.Greece, dialCode: '30', flag: '🇬🇷'}, + {name: 'Greenland', iso2: CountryISO.Greenland, dialCode: '299', flag: '🇬🇱'}, + {name: 'Grenada', iso2: CountryISO.Grenada, dialCode: '1473', flag: '🇬🇩'}, + {name: 'Guadeloupe', iso2: CountryISO.Guadeloupe, dialCode: '590', flag: '🇬🇵'}, + {name: 'Guam', iso2: CountryISO.Guam, dialCode: '1', flag: '🇬🇺'}, + {name: 'Guatemala', iso2: CountryISO.Guatemala, dialCode: '502', flag: '🇬🇹'}, + {name: 'Guernsey', iso2: CountryISO.Guernsey, dialCode: '44', flag: '🇬🇬'}, + {name: 'Guinea', iso2: CountryISO.Guinea, dialCode: '224', flag: '🇬🇳'}, + {name: 'Guinea-Bissau', iso2: CountryISO.GuineaBissau, dialCode: '245', flag: '🇬🇼'}, + {name: 'Guyana', iso2: CountryISO.Guyana, dialCode: '592', flag: '🇬🇾'}, + {name: 'Haiti', iso2: CountryISO.Haiti, dialCode: '509', flag: '🇭🇹'}, + {name: 'Honduras', iso2: CountryISO.Honduras, dialCode: '504', flag: '🇭🇳'}, + {name: 'Hong Kong', iso2: CountryISO.HongKong, dialCode: '852', flag: '🇭🇰'}, + {name: 'Hungary', iso2: CountryISO.Hungary, dialCode: '36', flag: '🇭🇺'}, + {name: 'Iceland', iso2: CountryISO.Iceland, dialCode: '354', flag: '🇮🇸'}, + {name: 'India', iso2: CountryISO.India, dialCode: '91', flag: '🇮🇳'}, + {name: 'Indonesia', iso2: CountryISO.Indonesia, dialCode: '62', flag: '🇮🇩'}, + {name: 'Iran', iso2: CountryISO.Iran, dialCode: '98', flag: '🇮🇷'}, + {name: 'Iraq', iso2: CountryISO.Iraq, dialCode: '964', flag: '🇮🇶'}, + {name: 'Ireland', iso2: CountryISO.Ireland, dialCode: '353', flag: '🇮🇪'}, + {name: 'Isle of Man', iso2: CountryISO.IsleOfMan, dialCode: '44', flag: '🇮🇲'}, + {name: 'Israel', iso2: CountryISO.Israel, dialCode: '972', flag: '🇮🇱'}, + {name: 'Italy', iso2: CountryISO.Italy, dialCode: '39', flag: '🇮🇹'}, + {name: 'Jamaica', iso2: CountryISO.Jamaica, dialCode: '1', flag: '🇯🇲'}, + {name: 'Japan', iso2: CountryISO.Japan, dialCode: '81', flag: '🇯🇵'}, + {name: 'Jersey', iso2: CountryISO.Jersey, dialCode: '44', flag: '🇯🇪'}, + {name: 'Jordan', iso2: CountryISO.Jordan, dialCode: '962', flag: '🇯🇴'}, + {name: 'Kazakhstan', iso2: CountryISO.Kazakhstan, dialCode: '7', flag: '🇰🇿'}, + {name: 'Kenya', iso2: CountryISO.Kenya, dialCode: '254', flag: '🇰🇪'}, + {name: 'Kiribati', iso2: CountryISO.Kiribati, dialCode: '686', flag: '🇰🇮'}, + {name: 'Kosovo', iso2: CountryISO.Kosovo, dialCode: '383', flag: '🇽🇰'}, + {name: 'Kuwait', iso2: CountryISO.Kuwait, dialCode: '965', flag: '🇰🇼'}, + {name: 'Kyrgyzstan', iso2: CountryISO.Kyrgyzstan, dialCode: '996', flag: '🇰🇬'}, + {name: 'Laos', iso2: CountryISO.Laos, dialCode: '856', flag: '🇱🇦'}, + {name: 'Latvia', iso2: CountryISO.Latvia, dialCode: '371', flag: '🇱🇻'}, + {name: 'Lebanon', iso2: CountryISO.Lebanon, dialCode: '961', flag: '🇱🇧'}, + {name: 'Lesotho', iso2: CountryISO.Lesotho, dialCode: '266', flag: '🇱🇸'}, + {name: 'Liberia', iso2: CountryISO.Liberia, dialCode: '231', flag: '🇱🇷'}, + {name: 'Libya', iso2: CountryISO.Libya, dialCode: '218', flag: '🇱🇾'}, + {name: 'Liechtenstein', iso2: CountryISO.Liechtenstein, dialCode: '423', flag: '🇱🇮'}, + {name: 'Lithuania', iso2: CountryISO.Lithuania, dialCode: '370', flag: '🇱🇹'}, + {name: 'Luxembourg', iso2: CountryISO.Luxembourg, dialCode: '352', flag: '🇱🇺'}, + {name: 'Macau', iso2: CountryISO.Macau, dialCode: '853', flag: '🇲🇴'}, + {name: 'Macedonia', iso2: CountryISO.Macedonia, dialCode: '389', flag: '🇲🇰'}, + {name: 'Madagascar', iso2: CountryISO.Madagascar, dialCode: '261', flag: '🇲🇬'}, + {name: 'Malawi', iso2: CountryISO.Malawi, dialCode: '265', flag: '🇲🇼'}, + {name: 'Malaysia', iso2: CountryISO.Malaysia, dialCode: '60', flag: '🇲🇾'}, + {name: 'Maldives', iso2: CountryISO.Maldives, dialCode: '960', flag: '🇲🇻'}, + {name: 'Mali', iso2: CountryISO.Mali, dialCode: '223', flag: '🇲🇱'}, + {name: 'Malta', iso2: CountryISO.Malta, dialCode: '356', flag: '🇲🇹'}, + {name: 'Marshall Islands', iso2: CountryISO.MarshallIslands, dialCode: '692', flag: '🇲🇭'}, + {name: 'Martinique', iso2: CountryISO.Martinique, dialCode: '596', flag: '🇲🇶'}, + {name: 'Mauritania', iso2: CountryISO.Mauritania, dialCode: '222', flag: '🇲🇷'}, + {name: 'Mauritius', iso2: CountryISO.Mauritius, dialCode: '230', flag: '🇲🇺'}, + {name: 'Mayotte', iso2: CountryISO.Mayotte, dialCode: '262', flag: '🇾🇹'}, + {name: 'Mexico', iso2: CountryISO.Mexico, dialCode: '52', flag: '🇲🇽'}, + {name: 'Micronesia', iso2: CountryISO.Micronesia, dialCode: '691', flag: '🇫🇲'}, + {name: 'Moldova', iso2: CountryISO.Moldova, dialCode: '373', flag: '🇲🇩'}, + {name: 'Monaco', iso2: CountryISO.Monaco, dialCode: '377', flag: '🇲🇨'}, + {name: 'Mongolia', iso2: CountryISO.Mongolia, dialCode: '976', flag: '🇲🇳'}, + {name: 'Montenegro', iso2: CountryISO.Montenegro, dialCode: '382', flag: '🇲🇪'}, + {name: 'Montserrat', iso2: CountryISO.Montserrat, dialCode: '1', flag: '🇲🇸'}, + {name: 'Morocco', iso2: CountryISO.Morocco, dialCode: '212', flag: '🇲🇦'}, + {name: 'Mozambique', iso2: CountryISO.Mozambique, dialCode: '258', flag: '🇲🇿'}, + {name: 'Myanmar', iso2: CountryISO.Myanmar, dialCode: '95', flag: '🇲🇲'}, + {name: 'Namibia', iso2: CountryISO.Namibia, dialCode: '264', flag: '🇳🇦'}, + {name: 'Nauru', iso2: CountryISO.Nauru, dialCode: '674', flag: '🇳🇷'}, + {name: 'Nepal', iso2: CountryISO.Nepal, dialCode: '977', flag: '🇳🇵'}, + {name: 'Netherlands', iso2: CountryISO.Netherlands, dialCode: '31', flag: '🇳🇱'}, + {name: 'New Caledonia', iso2: CountryISO.NewCaledonia, dialCode: '687', flag: '🇳🇨'}, + {name: 'New Zealand', iso2: CountryISO.NewZealand, dialCode: '64', flag: '🇳🇿'}, + {name: 'Nicaragua', iso2: CountryISO.Nicaragua, dialCode: '505', flag: '🇳🇮'}, + {name: 'Niger', iso2: CountryISO.Niger, dialCode: '227', flag: '🇳🇪'}, + {name: 'Nigeria', iso2: CountryISO.Nigeria, dialCode: '234', flag: '🇳🇬'}, + {name: 'Niue', iso2: CountryISO.Niue, dialCode: '683', flag: '🇳🇺'}, + {name: 'Norfolk Island', iso2: CountryISO.NorfolkIsland, dialCode: '672', flag: '🇳🇫'}, + {name: 'North Korea', iso2: CountryISO.NorthKorea, dialCode: '850', flag: '🇰🇵'}, + {name: 'Northern Mariana Islands', iso2: CountryISO.NorthernMarianaIslands, dialCode: '1670', flag: '🇲🇵'}, + {name: 'Norway', iso2: CountryISO.Norway, dialCode: '47', flag: '🇳🇴'}, + {name: 'Oman', iso2: CountryISO.Oman, dialCode: '968', flag: '🇴🇲'}, + {name: 'Pakistan', iso2: CountryISO.Pakistan, dialCode: '92', flag: '🇵🇰'}, + {name: 'Palau', iso2: CountryISO.Palau, dialCode: '680', flag: '🇵🇼'}, + {name: 'Palestine', iso2: CountryISO.Palestine, dialCode: '970', flag: '🇵🇸'}, + {name: 'Panama', iso2: CountryISO.Panama, dialCode: '507', flag: '🇵🇦'}, + {name: 'Papua New Guinea', iso2: CountryISO.PapuaNewGuinea, dialCode: '675', flag: '🇵🇬'}, + {name: 'Paraguay', iso2: CountryISO.Paraguay, dialCode: '595', flag: '🇵🇾'}, + {name: 'Peru', iso2: CountryISO.Peru, dialCode: '51', flag: '🇵🇪'}, + {name: 'Philippines', iso2: CountryISO.Philippines, dialCode: '63', flag: '🇵🇭'}, + {name: 'Poland', iso2: CountryISO.Poland, dialCode: '48', flag: '🇵🇱'}, + {name: 'Portugal', iso2: CountryISO.Portugal, dialCode: '351', flag: '🇵🇹'}, + {name: 'Puerto Rico', iso2: CountryISO.PuertoRico, dialCode: '1', flag: '🇵🇷'}, + {name: 'Qatar', iso2: CountryISO.Qatar, dialCode: '974', flag: '🇶🇦'}, + {name: 'Réunion', iso2: CountryISO.Réunion, dialCode: '262', flag: '🇷🇪'}, + {name: 'Romania', iso2: CountryISO.Romania, dialCode: '40', flag: '🇷🇴'}, + {name: 'Russia', iso2: CountryISO.Russia, dialCode: '7', flag: '🇷🇺'}, + {name: 'Rwanda', iso2: CountryISO.Rwanda, dialCode: '250', flag: '🇷🇼'}, + {name: 'Saint Barthélemy', iso2: CountryISO.SaintBarthélemy, dialCode: '590', flag: '🇧🇱'}, + {name: 'Saint Helena', iso2: CountryISO.SaintHelena, dialCode: '290', flag: '🇸🇭'}, + {name: 'Saint Kitts and Nevis', iso2: CountryISO.SaintKittsAndNevis, dialCode: '1869', flag: '🇰🇳'}, + {name: 'Saint Lucia', iso2: CountryISO.SaintLucia, dialCode: '1', flag: '🇱🇨'}, + {name: 'Saint Martin', iso2: CountryISO.SaintMartin, dialCode: '590', flag: '🇲🇫'}, + {name: 'Saint Pierre and Miquelon', iso2: CountryISO.SaintPierreAndMiquelon, dialCode: '508', flag: '🇵🇲'}, + {name: 'Saint Vincent and the Grenadines', iso2: CountryISO.SaintVincentAndTheGrenadines, dialCode: '1', flag: '🇻🇨'}, + {name: 'Samoa', iso2: CountryISO.Samoa, dialCode: '685', flag: '🇼🇸'}, + {name: 'San Marino', iso2: CountryISO.SanMarino, dialCode: '378', flag: '🇸🇲'}, + {name: 'São Tomé and Príncipe', iso2: CountryISO.SãoToméAndPríncipe, dialCode: '239', flag: '🇸🇹'}, + {name: 'Saudi Arabia', iso2: CountryISO.SaudiArabia, dialCode: '966', flag: '🇸🇦'}, + {name: 'Senegal', iso2: CountryISO.Senegal, dialCode: '221', flag: '🇸🇳'}, + {name: 'Serbia', iso2: CountryISO.Serbia, dialCode: '381', flag: '🇷🇸'}, + {name: 'Seychelles', iso2: CountryISO.Seychelles, dialCode: '248', flag: '🇸🇨'}, + {name: 'Sierra Leone', iso2: CountryISO.SierraLeone, dialCode: '232', flag: '🇸🇱'}, + {name: 'Singapore', iso2: CountryISO.Singapore, dialCode: '65', flag: '🇸🇬'}, + {name: 'Sint Maarten', iso2: CountryISO.SintMaarten, dialCode: '1', flag: '🇸🇽'}, + {name: 'Slovakia', iso2: CountryISO.Slovakia, dialCode: '421', flag: '🇸🇰'}, + {name: 'Slovenia', iso2: CountryISO.Slovenia, dialCode: '386', flag: '🇸🇮'}, + {name: 'Solomon Islands', iso2: CountryISO.SolomonIslands, dialCode: '677', flag: '🇸🇧'}, + {name: 'Somalia', iso2: CountryISO.Somalia, dialCode: '252', flag: '🇸🇴'}, + {name: 'South Africa', iso2: CountryISO.SouthAfrica, dialCode: '27', flag: '🇿🇦'}, + {name: 'South Korea', iso2: CountryISO.SouthKorea, dialCode: '82', flag: '🇰🇷'}, + {name: 'South Sudan', iso2: CountryISO.SouthSudan, dialCode: '211', flag: '🇸🇸'}, + {name: 'Spain', iso2: CountryISO.Spain, dialCode: '34', flag: '🇪🇸'}, + {name: 'Sri Lanka', iso2: CountryISO.SriLanka, dialCode: '94', flag: '🇱🇰'}, + {name: 'Sudan', iso2: CountryISO.Sudan, dialCode: '249', flag: '🇸🇩'}, + {name: 'Suriname: ', iso2: CountryISO.Suriname, dialCode: '597', flag: '🇸🇷'}, + {name: 'Svalbard and Jan Mayen', iso2: CountryISO.SvalbardAndJanMayen, dialCode: '47', flag: '🇸🇯'}, + {name: 'Swaziland', iso2: CountryISO.Swaziland, dialCode: '268', flag: '🇸🇿'}, + {name: 'Sweden', iso2: CountryISO.Sweden, dialCode: '46', flag: '🇸🇪'}, + {name: 'Switzerland', iso2: CountryISO.Switzerland, dialCode: '41', flag: '🇨🇭'}, + {name: 'Syria', iso2: CountryISO.Syria, dialCode: '963', flag: '🇸🇾'}, + {name: 'Taiwan', iso2: CountryISO.Taiwan, dialCode: '886', flag: '🇹🇼'}, + {name: 'Tajikistan', iso2: CountryISO.Tajikistan, dialCode: '992', flag: '🇹🇯'}, + {name: 'Tanzania', iso2: CountryISO.Tanzania, dialCode: '255', flag: '🇹🇿'}, + {name: 'Thailand', iso2: CountryISO.Thailand, dialCode: '66', flag: '🇹🇭'}, + {name: 'Timor-Leste', iso2: CountryISO.TimorLeste, dialCode: '670', flag: '🇹🇱'}, + {name: 'Togo', iso2: CountryISO.Togo, dialCode: '228', flag: '🇹🇬'}, + {name: 'Tokelau', iso2: CountryISO.Tokelau, dialCode: '690', flag: '🇹🇰'}, + {name: 'Tonga', iso2: CountryISO.Tonga, dialCode: '676', flag: '🇹🇴'}, + {name: 'Trinidad and Tobago', iso2: CountryISO.TrinidadAndTobago, dialCode: '1', flag: '🇹🇹'}, + {name: 'Tunisia', iso2: CountryISO.Tunisia, dialCode: '216', flag: '🇹🇳'}, + {name: 'Turkey', iso2: CountryISO.Turkey, dialCode: '90', flag: '🇹🇷'}, + {name: 'Turkmenistan', iso2: CountryISO.Turkmenistan, dialCode: '993', flag: '🇹🇲'}, + {name: 'Turks and Caicos Islands', iso2: CountryISO.TurksAndCaicosIslands, dialCode: '1649', flag: '🇹🇨'}, + {name: 'Tuvalu', iso2: CountryISO.Tuvalu, dialCode: '688', flag: '🇹🇻'}, + {name: 'U.S. Virgin Islands', iso2: CountryISO.USVirginIslands, dialCode: '1', flag: '🇻🇮'}, + {name: 'Uganda', iso2: CountryISO.Uganda, dialCode: '256', flag: '🇺🇬'}, + {name: 'Ukraine', iso2: CountryISO.Ukraine, dialCode: '380', flag: '🇺🇦'}, + {name: 'United Arab Emirates', iso2: CountryISO.UnitedArabEmirates, dialCode: '971', flag: '🇦🇪'}, + {name: 'United Kingdom', iso2: CountryISO.UnitedKingdom, dialCode: '44', flag: '🇬🇧'}, + {name: 'United States', iso2: CountryISO.UnitedStates, dialCode: '1', flag: '🇺🇸'}, + {name: 'Uruguay', iso2: CountryISO.Uruguay, dialCode: '598', flag: '🇺🇾'}, + {name: 'Uzbekistan', iso2: CountryISO.Uzbekistan, dialCode: '998', flag: '🇺🇿'}, + {name: 'Vanuatu', iso2: CountryISO.Vanuatu, dialCode: '678', flag: '🇻🇺'}, + {name: 'Vatican City', iso2: CountryISO.VaticanCity, dialCode: '39', flag: '🇻🇦'}, + {name: 'Venezuela', iso2: CountryISO.Venezuela, dialCode: '58', flag: '🇻🇪'}, + {name: 'Vietnam', iso2: CountryISO.Vietnam, dialCode: '84', flag: '🇻🇳'}, + {name: 'Wallis and Futuna', iso2: CountryISO.WallisAndFutuna, dialCode: '681', flag: '🇼🇫'}, + {name: 'Western Sahara', iso2: CountryISO.WesternSahara, dialCode: '212', flag: '🇪🇭'}, + {name: 'Yemen', iso2: CountryISO.Yemen, dialCode: '967', flag: '🇾🇪'}, + {name: 'Zambia', iso2: CountryISO.Zambia, dialCode: '260', flag: '🇿🇲'}, + {name: 'Zimbabwe', iso2: CountryISO.Zimbabwe, dialCode: '263', flag: '🇿🇼'}, + {name: 'Åland Islands', iso2: CountryISO.ÅlandIslands, dialCode: '358', flag: '🇦🇽'} ]; } From afc3820b79a1c54330fb6d30e1808e7d49de4b34 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Mon, 13 Jun 2022 16:48:15 +0300 Subject: [PATCH 206/262] UI: Change placeholder --- ui-ngx/src/app/shared/components/phone-input.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/shared/components/phone-input.component.html b/ui-ngx/src/app/shared/components/phone-input.component.html index 4b81391b5f..039f52a272 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.html +++ b/ui-ngx/src/app/shared/components/phone-input.component.html @@ -28,12 +28,12 @@
- phone-input.phone-input-label + security.2fa.dialog.sms-step-label Date: Mon, 13 Jun 2022 18:06:32 +0300 Subject: [PATCH 207/262] UI: Fixed vulnerabilities in tb-web-ui --- msa/web-ui/package.json | 21 +- msa/web-ui/pom.xml | 4 +- msa/web-ui/yarn.lock | 966 ++++++++++++++++++++++------------------ 3 files changed, 537 insertions(+), 454 deletions(-) diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 1cd252e0de..81e0663613 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -13,14 +13,14 @@ }, "dependencies": { "compression": "^1.7.4", - "config": "^3.3.1", + "config": "^3.3.7", "connect-history-api-fallback": "^1.6.0", - "express": "^4.17.1", + "express": "^4.18.1", "http": "0.0.0", "http-proxy": "^1.18.1", - "js-yaml": "^3.14.0", - "winston": "^3.3.3", - "winston-daily-rotate-file": "^4.5.0" + "js-yaml": "^4.1.0", + "winston": "^3.7.2", + "winston-daily-rotate-file": "^4.7.1" }, "nyc": { "exclude": [ @@ -31,13 +31,18 @@ ] }, "devDependencies": { - "fs-extra": "^10.0.0", - "nodemon": "^2.0.12", - "pkg": "^5.3.1" + "fs-extra": "^10.1.0", + "nodemon": "^2.0.16", + "pkg": "^5.7.0" }, "pkg": { "assets": [ "node_modules/config/**/*.*" ] + }, + "resolutions": { + "color-string": "^1.5.5", + "follow-redirects": "^1.14.8", + "minimist": "^1.2.6" } } diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index f04b339a8e..c949555d46 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -80,8 +80,8 @@ install-node-and-yarn - v12.16.1 - v1.22.4 + v16.13.1 + v1.22.17 diff --git a/msa/web-ui/yarn.lock b/msa/web-ui/yarn.lock index 2c7012b515..2204573d19 100644 --- a/msa/web-ui/yarn.lock +++ b/msa/web-ui/yarn.lock @@ -2,25 +2,29 @@ # yarn lockfile v1 -"@babel/helper-validator-identifier@^7.12.11": - version "7.14.9" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.9.tgz#6654d171b2024f6d8ee151bf2509699919131d48" - integrity sha512-pQYxPY0UP6IHISRitNe8bsijHex4TWZXi2HwKVsjPiltzlhse2znVcm9Ace510VT1kxIHjGJCZZQBX2gJDbo0g== - -"@babel/parser@7.13.13": - version "7.13.13" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.13.tgz#42f03862f4aed50461e543270916b47dd501f0df" - integrity sha512-OhsyMrqygfk5v8HmWwOzlYjJrtLaFhF34MrfG/Z73DgYCI6ojNUTUp2TYbtnjo8PegeJp12eamsNettCQjKjVw== - -"@babel/types@7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.12.tgz#edbf99208ef48852acdff1c8a681a1e4ade580cd" - integrity sha512-K4nY2xFN4QMvQwkQ+zmBDp6ANMbVNw6BbxWmYA4qNjhR9W+Lj/8ky5MEY2Me5r+B2c6/v6F53oMndG+f9s3IiA== - dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - lodash "^4.17.19" +"@babel/helper-validator-identifier@^7.16.7": + version "7.16.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" + integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== + +"@babel/parser@7.17.10": + version "7.17.10" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.10.tgz#873b16db82a8909e0fbd7f115772f4b739f6ce78" + integrity sha512-n2Q6i+fnJqzOaq2VkdXxy2TCPCWQZHiCo0XqmrCvDWcZQKRyZzYi4Z0yxlBuN0w+r2ZHmre+Q087DSrw3pbJDQ== + +"@babel/types@7.17.10": + version "7.17.10" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.17.10.tgz#d35d7b4467e439fcf06d195f8100e0fea7fc82c4" + integrity sha512-9O26jG0mBYfGkUYCYZRnBwbVLd1UZOICEr2Em6InB6jVfsAv1GKgwXHmrSg+WFWDmeKTA6vyTZiN8tCSM5Oo3A== + dependencies: + "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@colors/colors@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" + integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + "@dabh/diagnostics@^2.0.2": version "2.0.2" resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31" @@ -73,7 +77,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -accepts@~1.3.5, accepts@~1.3.7: +accepts@~1.3.5: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== @@ -81,6 +85,14 @@ accepts@~1.3.5, accepts@~1.3.7: mime-types "~2.1.24" negotiator "0.6.2" +accepts@~1.3.8: + version "1.3.8" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" + integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== + dependencies: + mime-types "~2.1.34" + negotiator "0.6.3" + agent-base@6: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -115,6 +127,11 @@ ansi-regex@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + ansi-styles@^4.0.0: version "4.3.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" @@ -130,10 +147,10 @@ ansi-styles@^4.1.0: "@types/color-name" "^1.1.1" color-convert "^2.0.1" -anymatch@~3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.1.tgz#c55ecf02185e2469259399310c173ce31233b142" - integrity sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg== +anymatch@~3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" + integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== dependencies: normalize-path "^3.0.0" picomatch "^2.0.4" @@ -151,12 +168,10 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" -argparse@^1.0.7: - version "1.0.10" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" - integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== - dependencies: - sprintf-js "~1.0.2" +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== array-flatten@1.1.1: version "1.1.1" @@ -168,10 +183,10 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -async@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== +async@^3.2.3: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== at-least-node@^1.0.0: version "1.0.0" @@ -202,35 +217,37 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" -body-parser@1.19.0: - version "1.19.0" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" - integrity sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw== +body-parser@1.20.0: + version "1.20.0" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.0.tgz#3de69bd89011c11573d7bfee6a64f11b6bd27cc5" + integrity sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg== dependencies: - bytes "3.1.0" + bytes "3.1.2" content-type "~1.0.4" debug "2.6.9" - depd "~1.1.2" - http-errors "1.7.2" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.7.0" - raw-body "2.4.0" - type-is "~1.6.17" + on-finished "2.4.1" + qs "6.10.3" + raw-body "2.5.1" + type-is "~1.6.18" + unpipe "1.0.0" -boxen@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" - integrity sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ== +boxen@^5.0.0: + version "5.1.2" + resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" + integrity sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ== dependencies: ansi-align "^3.0.0" - camelcase "^5.3.1" - chalk "^3.0.0" - cli-boxes "^2.2.0" - string-width "^4.1.0" - term-size "^2.1.0" - type-fest "^0.8.1" + camelcase "^6.2.0" + chalk "^4.1.0" + cli-boxes "^2.2.1" + string-width "^4.2.2" + type-fest "^0.20.2" widest-line "^3.1.0" + wrap-ansi "^7.0.0" brace-expansion@^1.1.7: version "1.1.11" @@ -240,7 +257,7 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -braces@^3.0.1, braces@~3.0.2: +braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -260,10 +277,10 @@ bytes@3.0.0: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" integrity sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg= -bytes@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" - integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== +bytes@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" + integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== cacheable-request@^6.0.0: version "6.1.0" @@ -278,20 +295,20 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" -camelcase@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" - integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== - -chalk@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" - integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" + function-bind "^1.1.1" + get-intrinsic "^1.0.2" -chalk@^4.1.0: +camelcase@^6.2.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== + +chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -299,20 +316,20 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chokidar@^3.2.2: - version "3.4.2" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.2.tgz#38dc8e658dec3809741eb3ef7bb0a47fe424232d" - integrity sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A== +chokidar@^3.5.2: + version "3.5.3" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" + integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== dependencies: - anymatch "~3.1.1" + anymatch "~3.1.2" braces "~3.0.2" - glob-parent "~5.1.0" + glob-parent "~5.1.2" is-binary-path "~2.1.0" is-glob "~4.0.1" normalize-path "~3.0.0" - readdirp "~3.4.0" + readdirp "~3.6.0" optionalDependencies: - fsevents "~2.1.2" + fsevents "~2.3.2" chownr@^1.1.1: version "1.1.4" @@ -324,10 +341,10 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== -cli-boxes@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.0.tgz#538ecae8f9c6ca508e3c3c95b453fe93cb4c168d" - integrity sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w== +cli-boxes@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" + integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== cliui@^7.0.2: version "7.0.4" @@ -374,10 +391,10 @@ color-name@^1.0.0, color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -color-string@^1.5.2: - version "1.5.3" - resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.3.tgz#c9bbc5f01b58b5492f3d6857459cb6590ce204cc" - integrity sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw== +color-string@^1.5.2, color-string@^1.5.5: + version "1.9.1" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" + integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== dependencies: color-name "^1.0.0" simple-swizzle "^0.2.2" @@ -390,11 +407,6 @@ color@3.0.x: color-convert "^1.9.1" color-string "^1.5.2" -colors@^1.2.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== - colorspace@1.1.x: version "1.1.2" resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5" @@ -428,10 +440,10 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= -config@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/config/-/config-3.3.1.tgz#b6a70e2908a43b98ed20be7e367edf0cc8ed5a19" - integrity sha512-+2/KaaaAzdwUBE3jgZON11L1ggLLhpf2FsGrfqYFHZW22ySGv/HqYIXrBwKKvn+XZh1UBUjHwAcrfsSkSygT+Q== +config@^3.3.7: + version "3.3.7" + resolved "https://registry.yarnpkg.com/config/-/config-3.3.7.tgz#4310410dc2bf4e0effdca21a12a4035860a24ee4" + integrity sha512-mX/n7GKDYZMqvvkY6e6oBY49W8wxdmQt+ho/5lhwFDXqQW9gI+Ahp8EKp8VAbISPnmf2+Bv5uZK7lKXZ6pf1aA== dependencies: json5 "^2.1.1" @@ -457,12 +469,12 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0: resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= -content-disposition@0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" - integrity sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g== +content-disposition@0.5.4: + version "0.5.4" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" + integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== dependencies: - safe-buffer "5.1.2" + safe-buffer "5.2.1" content-type@~1.0.4: version "1.0.4" @@ -474,10 +486,10 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= -cookie@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba" - integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg== +cookie@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" + integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== core-util-is@~1.0.0: version "1.0.2" @@ -489,7 +501,7 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== -debug@2.6.9, debug@^2.2.0: +debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -503,10 +515,10 @@ debug@4: dependencies: ms "2.1.2" -debug@^3.2.6: - version "3.2.6" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" - integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== dependencies: ms "^2.1.1" @@ -544,15 +556,15 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= -depd@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" - integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= +depd@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-libc@^1.0.3: version "1.0.3" @@ -637,7 +649,7 @@ escodegen@^2.0.0: optionalDependencies: source-map "~0.6.1" -esprima@^4.0.0, esprima@^4.0.1: +esprima@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -667,64 +679,59 @@ expand-template@^2.0.3: resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== -express@^4.17.1: - version "4.17.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" - integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g== +express@^4.18.1: + version "4.18.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.18.1.tgz#7797de8b9c72c857b9cd0e14a5eea80666267caf" + integrity sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q== dependencies: - accepts "~1.3.7" + accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.19.0" - content-disposition "0.5.3" + body-parser "1.20.0" + content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.4.0" + cookie "0.5.0" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.2" + depd "2.0.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "~1.1.2" + finalhandler "1.2.0" fresh "0.5.2" + http-errors "2.0.0" merge-descriptors "1.0.1" methods "~1.1.2" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" path-to-regexp "0.1.7" - proxy-addr "~2.0.5" - qs "6.7.0" + proxy-addr "~2.0.7" + qs "6.10.3" range-parser "~1.2.1" - safe-buffer "5.1.2" - send "0.17.1" - serve-static "1.14.1" - setprototypeof "1.1.1" - statuses "~1.5.0" + safe-buffer "5.2.1" + send "0.18.0" + serve-static "1.15.0" + setprototypeof "1.2.0" + statuses "2.0.1" type-is "~1.6.18" utils-merge "1.0.1" vary "~1.1.2" -fast-glob@^3.1.1: - version "3.2.4" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3" - integrity sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ== +fast-glob@^3.2.9: + version "3.2.11" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9" + integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.0" + glob-parent "^5.1.2" merge2 "^1.3.0" - micromatch "^4.0.2" - picomatch "^2.2.1" + micromatch "^4.0.4" fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fast-safe-stringify@^2.0.4: - version "2.0.7" - resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" - integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== - fastq@^1.6.0: version "1.8.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.8.0.tgz#550e1f9f59bbc65fe185cb6a9b4d95357107f481" @@ -737,12 +744,12 @@ fecha@^4.2.0: resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== -file-stream-rotator@^0.5.7: - version "0.5.7" - resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.5.7.tgz#868a2e5966f7640a17dd86eda0e4467c089f6286" - integrity sha512-VYb3HZ/GiAGUCrfeakO8Mp54YGswNUHvL7P09WQcXAJNSj3iQ5QraYSp3cIn1MUyw6uzfgN/EFOarCNa4JvUHQ== +file-stream-rotator@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz#007019e735b262bb6c6f0197e58e5c87cb96cec3" + integrity sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ== dependencies: - moment "^2.11.2" + moment "^2.29.1" fill-range@^7.0.1: version "7.0.1" @@ -751,17 +758,17 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== +finalhandler@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== dependencies: debug "2.6.9" encodeurl "~1.0.2" escape-html "~1.0.3" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" - statuses "~1.5.0" + statuses "2.0.1" unpipe "~1.0.0" fn.name@1.x.x: @@ -769,15 +776,15 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.0.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db" - integrity sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA== +follow-redirects@^1.0.0, follow-redirects@^1.14.8: + version "1.15.1" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" + integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== -forwarded@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" - integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ= +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== fresh@0.5.2: version "0.5.2" @@ -797,10 +804,10 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.0.0.tgz#9ff61b655dde53fb34a82df84bb214ce802e17c1" - integrity sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ== +fs-extra@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== dependencies: graceful-fs "^4.2.0" jsonfile "^6.0.1" @@ -816,10 +823,10 @@ fs-extra@^9.1.0: jsonfile "^6.0.1" universalify "^2.0.0" -fsevents@~2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" - integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fsevents@~2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== function-bind@^1.1.1: version "1.1.1" @@ -845,6 +852,15 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-intrinsic@^1.0.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz#336975123e05ad0b7ba41f152ee4aadbea6cf598" + integrity sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + get-stream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -864,30 +880,30 @@ github-from-package@0.0.0: resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= -glob-parent@^5.1.0, glob-parent@~5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" - integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ== +glob-parent@^5.1.2, glob-parent@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" -global-dirs@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.0.1.tgz#acdf3bb6685bcd55cb35e8a052266569e9469201" - integrity sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A== +global-dirs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" + integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== dependencies: - ini "^1.3.5" + ini "2.0.0" -globby@^11.0.3: - version "11.0.4" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5" - integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg== +globby@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" + integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== dependencies: array-union "^2.1.0" dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" + fast-glob "^3.2.9" + ignore "^5.2.0" + merge2 "^1.4.1" slash "^3.0.0" got@^9.6.0: @@ -922,6 +938,11 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" @@ -944,27 +965,16 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== -http-errors@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" - integrity sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - -http-errors@~1.7.2: - version "1.7.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" - integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== +http-errors@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: - depd "~1.1.2" + depd "2.0.0" inherits "2.0.4" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" + setprototypeof "1.2.0" + statuses "2.0.1" + toidentifier "1.0.1" http-proxy@^1.18.1: version "1.18.1" @@ -1005,10 +1015,10 @@ ignore-by-default@^1.0.1: resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" integrity sha1-SMptcvbGo68Aqa1K5odr44ieKwk= -ignore@^5.1.4: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== +ignore@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" + integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== import-lazy@^2.1.0: version "2.1.0" @@ -1020,17 +1030,17 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -inherits@2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= - inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@^1.3.5, ini@~1.3.0: +ini@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + +ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== @@ -1067,10 +1077,10 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.2.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.6.0.tgz#d7553b2526fe59b92ba3e40c8df757ec8a709e19" - integrity sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ== +is-core-module@2.9.0, is-core-module@^2.8.1: + version "2.9.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69" + integrity sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A== dependencies: has "^1.0.3" @@ -1103,18 +1113,18 @@ is-glob@^4.0.1, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-installed-globally@^0.3.1: - version "0.3.2" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.3.2.tgz#fd3efa79ee670d1187233182d5b0a1dd00313141" - integrity sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g== +is-installed-globally@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" + integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== dependencies: - global-dirs "^2.0.1" - is-path-inside "^3.0.1" + global-dirs "^3.0.0" + is-path-inside "^3.0.2" -is-npm@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d" - integrity sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig== +is-npm@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-5.0.0.tgz#43e8d65cc56e1b67f8d47262cf667099193f45a8" + integrity sha512-WW/rQLOazUq+ST/bCAVBp/2oMERWLsR7OrKyt052dNDk4DHcDE0/7QSXITlmi+VBcV13DfIbysG3tZJm5RfdBA== is-number@^7.0.0: version "7.0.0" @@ -1126,10 +1136,10 @@ is-obj@^2.0.0: resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== -is-path-inside@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" - integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== +is-path-inside@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== is-stream@^2.0.0: version "2.0.0" @@ -1151,13 +1161,12 @@ isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= -js-yaml@^3.14.0: - version "3.14.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482" - integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== dependencies: - argparse "^1.0.7" - esprima "^4.0.0" + argparse "^2.0.1" json-buffer@3.0.0: version "3.0.0" @@ -1192,7 +1201,7 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -latest-version@^5.0.0: +latest-version@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" integrity sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA== @@ -1207,20 +1216,15 @@ levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -lodash@^4.17.19: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -logform@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/logform/-/logform-2.2.0.tgz#40f036d19161fc76b68ab50fdc7fe495544492f2" - integrity sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg== +logform@^2.3.2, logform@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.4.0.tgz#131651715a17d50f09c2a2c1a524ff1a4164bcfe" + integrity sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw== dependencies: - colors "^1.2.1" - fast-safe-stringify "^2.0.4" + "@colors/colors" "1.5.0" fecha "^4.2.0" ms "^2.1.1" + safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: @@ -1257,7 +1261,7 @@ merge-descriptors@1.0.1: resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= -merge2@^1.3.0: +merge2@^1.3.0, merge2@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== @@ -1267,19 +1271,24 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= -micromatch@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259" - integrity sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q== +micromatch@^4.0.4: + version "4.0.5" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" + integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== dependencies: - braces "^3.0.1" - picomatch "^2.0.5" + braces "^3.0.2" + picomatch "^2.3.1" mime-db@1.44.0, "mime-db@>= 1.43.0 < 2": version "1.44.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + mime-types@~2.1.24: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -1287,6 +1296,13 @@ mime-types@~2.1.24: dependencies: mime-db "1.44.0" +mime-types@~2.1.34: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" @@ -1309,36 +1325,36 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6: + version "1.2.6" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -moment@^2.11.2: - version "2.27.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" - integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== +moment@^2.29.1: + version "2.29.3" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" + integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" - integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== - ms@2.1.2, ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + multistream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/multistream/-/multistream-4.1.0.tgz#7bf00dfd119556fbc153cff3de4c6d477909f5a8" @@ -1357,38 +1373,40 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== -node-abi@^2.7.0: +negotiator@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" + integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== + +node-abi@^2.21.0: version "2.30.1" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.30.1.tgz#c437d4b1fe0e285aaf290d45b45d4d7afedac4cf" integrity sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w== dependencies: semver "^5.4.1" -node-fetch@^2.6.1: - version "2.6.2" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.2.tgz#986996818b73785e47b1965cc34eb093a1d464d0" - integrity sha512-aLoxToI6RfZ+0NOjmWAgn9+LEd30YCkJKFSyWacNZdEKTit/ZMcKjGkTRo8uWEsnIb/hfKecNPEbln02PdWbcA== +node-fetch@^2.6.6: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" -nodemon@^2.0.12: - version "2.0.12" - resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.12.tgz#5dae4e162b617b91f1873b3bfea215dd71e144d5" - integrity sha512-egCTmNZdObdBxUBw6ZNwvZ/xzk24CKRs5K6d+5zbmrMr7rOpPmfPeF6OxM3DDpaRx331CQRFEktn+wrFFfBSOA== +nodemon@^2.0.16: + version "2.0.16" + resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.16.tgz#d71b31bfdb226c25de34afea53486c8ef225fdef" + integrity sha512-zsrcaOfTWRuUzBn3P44RDliLlp263Z/76FPoHFr3cFFkOz0lTPAcIw8dCzfdVIx/t3AtDYCZRCDkoCojJqaG3w== dependencies: - chokidar "^3.2.2" - debug "^3.2.6" + chokidar "^3.5.2" + debug "^3.2.7" ignore-by-default "^1.0.1" minimatch "^3.0.4" - pstree.remy "^1.1.7" + pstree.remy "^1.1.8" semver "^5.7.1" supports-color "^5.5.0" touch "^3.1.0" - undefsafe "^2.0.3" - update-notifier "^4.1.0" - -noop-logger@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" - integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI= + undefsafe "^2.0.5" + update-notifier "^5.1.0" nopt@~1.0.10: version "1.0.10" @@ -1432,10 +1450,15 @@ object-hash@^2.0.1: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea" integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= +object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + +on-finished@2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" @@ -1495,10 +1518,10 @@ parseurl@~1.3.3: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -path-parse@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" - integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-to-regexp@0.1.7: version "0.1.7" @@ -1510,49 +1533,54 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1: +picomatch@^2.0.4, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== -pkg-fetch@3.2.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.2.2.tgz#33f391eb176c1844e93189a32f2279b36a1ec949" - integrity sha512-bLhFNT4cNnONxzbHo1H2mCCKuQkCR4dgQtv0gUZnWtp8TDP0v0UAXKHG7DXhAoTC5IYP3slLsFJtIda9ksny8g== +picomatch@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + +pkg-fetch@3.4.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.4.1.tgz#be68bb9f7fdb0f6ed995abc518ab2e35aa64d2fd" + integrity sha512-fS4cdayCa1r4jHkOKGPJKnS9PEs6OWZst+s+m0+CmhmPZObMnxoRnf9T9yUWl+lzM2b5aJF7cnQIySCT7Hq8Dg== dependencies: - chalk "^4.1.0" + chalk "^4.1.2" fs-extra "^9.1.0" https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" + node-fetch "^2.6.6" progress "^2.0.3" semver "^7.3.5" + tar-fs "^2.1.1" yargs "^16.2.0" -pkg@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.3.1.tgz#8f81671613b9e5bb1d83c39b2eed4799e1e679fe" - integrity sha512-jT/sptM1ZG++FNk+jnJYNoWLDQXYd7hqpnBhd5j18SNW1jJzNYo55RahuCiD0KN0PX9mb53GWCqKM0ia/mJytA== +pkg@^5.7.0: + version "5.7.0" + resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.7.0.tgz#6422df05e8aa147764be6ef912921d0fa719ea95" + integrity sha512-PTiAjNq/CGAtK5qUBR6pjheqnipTFjeecgSgIKEcAOJA4GpmZeOZC8pMOoT0rfes5vHsmcFo7wbSRTAmXQurrg== dependencies: - "@babel/parser" "7.13.13" - "@babel/types" "7.13.12" - chalk "^4.1.0" + "@babel/parser" "7.17.10" + "@babel/types" "7.17.10" + chalk "^4.1.2" escodegen "^2.0.0" fs-extra "^9.1.0" - globby "^11.0.3" + globby "^11.1.0" into-stream "^6.0.0" - minimist "^1.2.5" + is-core-module "2.9.0" + minimist "^1.2.6" multistream "^4.1.0" - pkg-fetch "3.2.2" - prebuild-install "6.0.1" - progress "^2.0.3" - resolve "^1.20.0" + pkg-fetch "3.4.1" + prebuild-install "6.1.4" + resolve "^1.22.0" stream-meter "^1.0.4" - tslib "2.1.0" -prebuild-install@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-6.0.1.tgz#5902172f7a40eb67305b96c2a695db32636ee26d" - integrity sha512-7GOJrLuow8yeiyv75rmvZyeMGzl8mdEX5gY69d6a6bHWmiPevwqFw+tQavhK0EYMaSg3/KD24cWqeQv1EWsqDQ== +prebuild-install@6.1.4: + version "6.1.4" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-6.1.4.tgz#ae3c0142ad611d58570b89af4986088a4937e00f" + integrity sha512-Z4vpywnK1lBg+zdPCVCsKq0xO66eEV9rWo2zrROGGiRS4JtueBOdlB1FnY8lcy7JsUud/Q3ijUxyWN26Ika0vQ== dependencies: detect-libc "^1.0.3" expand-template "^2.0.3" @@ -1560,15 +1588,13 @@ prebuild-install@6.0.1: minimist "^1.2.3" mkdirp-classic "^0.5.3" napi-build-utils "^1.0.1" - node-abi "^2.7.0" - noop-logger "^0.1.1" + node-abi "^2.21.0" npmlog "^4.0.1" pump "^3.0.0" rc "^1.2.7" simple-get "^3.0.3" tar-fs "^2.0.0" tunnel-agent "^0.6.0" - which-pm-runs "^1.0.0" prelude-ls@~1.1.2: version "1.1.2" @@ -1590,15 +1616,15 @@ progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -proxy-addr@~2.0.5: - version "2.0.6" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf" - integrity sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw== +proxy-addr@~2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== dependencies: - forwarded "~0.1.2" + forwarded "0.2.0" ipaddr.js "1.9.1" -pstree.remy@^1.1.7: +pstree.remy@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w== @@ -1611,30 +1637,32 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" -pupa@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.0.1.tgz#dbdc9ff48ffbea4a26a069b6f9f7abb051008726" - integrity sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA== +pupa@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" + integrity sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A== dependencies: escape-goat "^2.0.0" -qs@6.7.0: - version "6.7.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" - integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== +qs@6.10.3: + version "6.10.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.3.tgz#d6cde1b2ffca87b5aa57889816c5f81535e22e8e" + integrity sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ== + dependencies: + side-channel "^1.0.4" range-parser@~1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332" - integrity sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q== +raw-body@2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: - bytes "3.1.0" - http-errors "1.7.2" + bytes "3.1.2" + http-errors "2.0.0" iconv-lite "0.4.24" unpipe "1.0.0" @@ -1670,10 +1698,10 @@ readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" -readdirp@~3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.4.0.tgz#9fdccdf9e9155805449221ac645e8303ab5b9ada" - integrity sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ== +readdirp@~3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" + integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" @@ -1701,13 +1729,14 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= -resolve@^1.20.0: - version "1.20.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" - integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== +resolve@^1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198" + integrity sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw== dependencies: - is-core-module "^2.2.0" - path-parse "^1.0.6" + is-core-module "^2.8.1" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" responselike@^1.0.2: version "1.0.2" @@ -1731,11 +1760,16 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@^5.0.1, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safe-stable-stringify@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73" + integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg== + "safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -1758,6 +1792,13 @@ semver@^6.0.0, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.3.4: + version "7.3.7" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" + integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== + dependencies: + lru-cache "^6.0.0" + semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" @@ -1765,44 +1806,53 @@ semver@^7.3.5: dependencies: lru-cache "^6.0.0" -send@0.17.1: - version "0.17.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" - integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== +send@0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== dependencies: debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" + depd "2.0.0" + destroy "1.2.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" - http-errors "~1.7.2" + http-errors "2.0.0" mime "1.6.0" - ms "2.1.1" - on-finished "~2.3.0" + ms "2.1.3" + on-finished "2.4.1" range-parser "~1.2.1" - statuses "~1.5.0" + statuses "2.0.1" -serve-static@1.14.1: - version "1.14.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" - integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== +serve-static@1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== dependencies: encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.17.1" + send "0.18.0" set-blocking@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= -setprototypeof@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" - integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.3" @@ -1840,20 +1890,15 @@ source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= - stack-trace@0.0.x: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= -"statuses@>= 1.5.0 < 2", statuses@~1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" - integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== stream-meter@^1.0.4: version "1.0.4" @@ -1906,6 +1951,15 @@ string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" +string-width@^4.2.2: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" @@ -1948,6 +2002,13 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" @@ -1967,7 +2028,12 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -tar-fs@^2.0.0: +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -1988,11 +2054,6 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" -term-size@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753" - integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw== - text-hex@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" @@ -2015,10 +2076,10 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -toidentifier@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" - integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +toidentifier@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" + integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== touch@^3.1.0: version "3.1.0" @@ -2027,16 +2088,16 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + triple-beam@^1.2.0, triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== -tslib@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" - integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== - tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -2051,12 +2112,12 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-fest@^0.8.1: - version "0.8.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" - integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -type-is@~1.6.17, type-is@~1.6.18: +type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== @@ -2071,12 +2132,10 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -undefsafe@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.3.tgz#6b166e7094ad46313b2202da7ecc2cd7cc6e7aae" - integrity sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A== - dependencies: - debug "^2.2.0" +undefsafe@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" + integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== unique-string@^2.0.0: version "2.0.0" @@ -2100,22 +2159,23 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= -update-notifier@^4.1.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.3.tgz#be86ee13e8ce48fb50043ff72057b5bd598e1ea3" - integrity sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A== +update-notifier@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" + integrity sha512-ItnICHbeMh9GqUy31hFPrD1kcuZ3rpxDZbf4KUDavXwS0bW5m7SLbDQpGX3UYr072cbrF5hFUs3r5tUsPwjfHw== dependencies: - boxen "^4.2.0" - chalk "^3.0.0" + boxen "^5.0.0" + chalk "^4.1.0" configstore "^5.0.1" has-yarn "^2.1.0" import-lazy "^2.1.0" is-ci "^2.0.0" - is-installed-globally "^0.3.1" - is-npm "^4.0.0" + is-installed-globally "^0.4.0" + is-npm "^5.0.0" is-yarn-global "^0.3.0" - latest-version "^5.0.0" - pupa "^2.0.1" + latest-version "^5.1.0" + pupa "^2.1.1" + semver "^7.3.4" semver-diff "^3.1.1" xdg-basedir "^4.0.0" @@ -2141,10 +2201,18 @@ vary@~1.1.2: resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= -which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" - integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" wide-align@^1.1.0: version "1.1.3" @@ -2160,17 +2228,17 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" -winston-daily-rotate-file@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.5.0.tgz#3914ac57c4bdae1138170bec85af0c2217b253b1" - integrity sha512-/HqeWiU48dzGqcrABRlxYWVMdL6l3uKCtFSJyrqK+E2rLnSFNsgYpvwx15EgTitBLNzH69lQd/+z2ASryV2aqw== +winston-daily-rotate-file@^4.7.1: + version "4.7.1" + resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz#f60a643af87f8867f23170d8cd87dbe3603a625f" + integrity sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA== dependencies: - file-stream-rotator "^0.5.7" + file-stream-rotator "^0.6.1" object-hash "^2.0.1" triple-beam "^1.3.0" - winston-transport "^4.2.0" + winston-transport "^4.4.0" -winston-transport@^4.2.0, winston-transport@^4.4.0: +winston-transport@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.4.0.tgz#17af518daa690d5b2ecccaa7acf7b20ca7925e59" integrity sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw== @@ -2178,20 +2246,30 @@ winston-transport@^4.2.0, winston-transport@^4.4.0: readable-stream "^2.3.7" triple-beam "^1.2.0" -winston@^3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/winston/-/winston-3.3.3.tgz#ae6172042cafb29786afa3d09c8ff833ab7c9170" - integrity sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw== +winston-transport@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa" + integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q== + dependencies: + logform "^2.3.2" + readable-stream "^3.6.0" + triple-beam "^1.3.0" + +winston@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.7.2.tgz#95b4eeddbec902b3db1424932ac634f887c400b1" + integrity sha512-QziIqtojHBoyzUOdQvQiar1DH0Xp9nF1A1y7NVy2DGEsz82SBDtOalS0ulTRGVT14xPX3WRWkCsdcJKqNflKng== dependencies: "@dabh/diagnostics" "^2.0.2" - async "^3.1.0" + async "^3.2.3" is-stream "^2.0.0" - logform "^2.2.0" + logform "^2.4.0" one-time "^1.0.0" readable-stream "^3.4.0" + safe-stable-stringify "^2.3.1" stack-trace "0.0.x" triple-beam "^1.3.0" - winston-transport "^4.4.0" + winston-transport "^4.5.0" word-wrap@~1.2.3: version "1.2.3" From 258417346f2d672d2d8b795f5a05b57fd6c96937 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Jun 2022 09:16:32 +0300 Subject: [PATCH 208/262] refactoring: customer - test start --- .../controller/AbstractNotifyEntityTest.java | 113 +++++++++++----- .../controller/BaseAlarmControllerTest.java | 22 +-- .../BaseCustomerControllerTest.java | 125 ++++++++++++++---- 3 files changed, 192 insertions(+), 68 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index a5827d6f6d..6fb7b93bae 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import lombok.extern.slf4j.Slf4j; import org.mockito.Mockito; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.cluster.TbClusterService; @@ -26,6 +27,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.model.ModelConstants; @@ -35,6 +38,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.thingsboard.server.service.entitiy.DefaultTbNotificationEntityService.edgeTypeByActionType; +@Slf4j public abstract class AbstractNotifyEntityTest extends AbstractWebTest { @SpyBean @@ -43,31 +47,53 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { @SpyBean protected AuditLogService auditLogService; - protected void testNotifyEntityOk(HasName entity, EntityId entityId, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType) { - testSendNotificationMsgToEdgeServiceOk(entityId, tenantId, actionType); - testLogEntityActionOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); - testPushMsgToRuleEngineOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + protected void testNotifyEntityOne(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, Object... additionalInfo) { + testSendNotificationMsgToEdgeServiceOne(entityId, tenantId, actionType); + testLogEntityActionOne(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); + testPushMsgToRuleEngineOne(originatorId, tenantId); Mockito.reset(tbClusterService, auditLogService); } - protected void testNotifyEntityDeleteOk(HasName entity, EntityId entityId, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType) { - Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdgeService(Mockito.any(), - Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.any()); - testLogEntityActionOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); - testPushMsgToRuleEngineOk(entity, originatorId, tenantId, customerId, userId, userName, actionType); + + protected void testNotifyEntityDeleteOneMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, Object... additionalInfo) { + testNotificationMsgToEdgeServiceNever(entityId); + testLogEntityActionOne(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); + testPushMsgToRuleEngineOne(entityId, tenantId); + testBroadcastEntityStateChangeEventOne(entityId, tenantId); + } - private void testNotifyEntityError(EntityId entityId, HasName entity, TenantId tenantId, - UserId userId, String userName, ActionType actionType, Exception exp, - Object... additionalInfo) { + protected void testNotifyEntityOneMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, Object... additionalInfo) { + testNotificationMsgToEdgeServiceNever(entityId); + testLogEntityActionOne(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); + testPushMsgToRuleEngineOne(originatorId, tenantId); + Mockito.reset(tbClusterService, auditLogService); + } + + protected void testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, Object... additionalInfo) { + testNotificationMsgToEdgeServiceNever(entityId); + testLogEntityActionOne(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); + testPushMsgToRuleEngineOne(originatorId, tenantId); + testBroadcastEntityStateChangeEventOne(entityId, tenantId); + Mockito.reset(tbClusterService, auditLogService); + } + + protected void testNotifyEntityError(HasName entity, TenantId tenantId, + UserId userId, String userName, ActionType actionType, Exception exp, + Object... additionalInfo) { CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); - EntityId entity_NULL_UUID = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(entity.getClass().toString().substring(entity.getClass().toString().lastIndexOf(".") + 1).toUpperCase(Locale.ENGLISH)), + EntityId entity_NULL_UUID = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(entity.getClass().toString() + .substring(entity.getClass().toString().lastIndexOf(".") + 1).toUpperCase(Locale.ENGLISH)), ModelConstants.NULL_UUID); - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceNever(entity_NULL_UUID); if (additionalInfo.length > 0) { Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), @@ -77,9 +103,9 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), - Mockito.any(exp.getClass()), Mockito.isNull()); + Mockito.any(exp.getClass())); } - Mockito.verify(tbClusterService, never()).pushMsgToRuleEngine(Mockito.any(), Mockito.any(entityId.getClass()), + Mockito.verify(tbClusterService, never()).pushMsgToRuleEngine(Mockito.any(), Mockito.any(entity_NULL_UUID.getClass()), Mockito.any(), Mockito.any()); Mockito.reset(tbClusterService, auditLogService); } @@ -106,27 +132,46 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any()); } - private void testLogEntityActionOk(HasName entity, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType) { - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), - Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), - Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); + private void testLogEntityActionOne(HasName entity, EntityId originatorId, TenantId tenantId, CustomerId customerId, + UserId userId, String userName, ActionType actionType, Object... additionalInfo) { + if (additionalInfo.length == 0) { + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), + Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), + Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); + } else { + String additionalInfoStr = extractParameter(String.class, 0, additionalInfo); + Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), + Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), + Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull(), Mockito.eq(additionalInfoStr)); + } } - private void testPushMsgToRuleEngineOk(HasName entity, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType) { - - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), - Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), - Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); + private void testPushMsgToRuleEngineOne(EntityId originatorId, TenantId tenantId) { + Mockito.verify(tbClusterService, times(1)).pushMsgToRuleEngine(Mockito.eq(tenantId), + Mockito.eq(originatorId), Mockito.any(TbMsg.class), Mockito.isNull()); } - private void testSendNotificationMsgToEdgeServiceOk(EntityId entityId, TenantId tenantId, ActionType actionType) { + private void testSendNotificationMsgToEdgeServiceOne(EntityId entityId, TenantId tenantId, ActionType actionType) { Mockito.verify(tbClusterService, times(1)).sendNotificationMsgToEdgeService(Mockito.eq(tenantId), Mockito.isNull(), Mockito.eq(entityId), Mockito.isNull(), Mockito.isNull(), Mockito.eq(edgeTypeByActionType(actionType))); } + private void testBroadcastEntityStateChangeEventOne(EntityId entityId, TenantId tenantId) { + Mockito.verify(tbClusterService, times(1)).broadcastEntityStateChangeEvent(Mockito.eq(tenantId), + Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); + } + + private T extractParameter(Class clazz, int index, Object... additionalInfo) { + T result = null; + if (additionalInfo != null && additionalInfo.length > index) { + Object paramObject = additionalInfo[index]; + if (clazz.isInstance(paramObject)) { + result = clazz.cast(paramObject); + } + } + return result; + } + + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java index 16a9f993c3..d82f4175b5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java @@ -65,7 +65,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); logout(); } @@ -78,7 +78,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); logout(); } @@ -91,7 +91,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); alarm.setSeverity(AlarmSeverity.MAJOR); @@ -99,7 +99,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); - testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.UPDATED); logout(); } @@ -112,7 +112,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); alarm.setSeverity(AlarmSeverity.MAJOR); @@ -120,7 +120,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); - testNotifyEntityOk(updatedAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), + testNotifyEntityOne(updatedAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED); logout(); } @@ -166,7 +166,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); - testNotifyEntityDeleteOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOneMsgToEdgeServiceNever(alarm, alarm.getId(), alarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.DELETED); logout(); } @@ -180,7 +180,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); - testNotifyEntityDeleteOk(alarm, alarm.getId(), alarm.getOriginator(), + testNotifyEntityOneMsgToEdgeServiceNever(alarm, alarm.getId(), alarm.getOriginator(), tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.DELETED); logout(); } @@ -229,7 +229,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOne(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_CLEAR); logout(); } @@ -246,7 +246,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOne(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR); logout(); } @@ -264,7 +264,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.ACTIVE_ACK, foundAlarm.getStatus()); - testNotifyEntityOk(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOne(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ACK); logout(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index a7f623d2e4..0aa81c2d18 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -25,14 +25,17 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; @@ -86,71 +89,164 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest public void testSaveCustomer() throws Exception { Customer customer = new Customer(); customer.setTitle("My customer"); + + Mockito.reset(tbClusterService, auditLogService); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); Assert.assertNotNull(savedCustomer); Assert.assertNotNull(savedCustomer.getId()); Assert.assertTrue(savedCustomer.getCreatedTime() > 0); Assert.assertEquals(customer.getTitle(), savedCustomer.getTitle()); + + testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), + tenantAdmin.getEmail(), ActionType.ADDED); + savedCustomer.setTitle("My new customer"); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/customer", savedCustomer, Customer.class); + testNotifyEntityOne(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); + Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); Assert.assertEquals(foundCustomer.getTitle(), savedCustomer.getTitle()); doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); + + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedCustomer.getId().getId().toString()); } @Test public void testSaveCustomerWithViolationOfValidation() throws Exception { Customer customer = new Customer(); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/customer", customer) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Customer title should be specified"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: empty title")); + customer.setTitle(RandomStringUtils.randomAlphabetic(300)); + doPost("/api/customer", customer).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad title")); + customer.setTitle("Normal title"); + customer.setEmail("invalid@mail"); + doPost("/api/customer", customer) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Invalid email address format 'invalid@mail'"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: invalid email")); + + customer.setEmail("normal@mail.com.ua"); + customer.setCity(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of city must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad City")); + customer.setCity("Normal city"); customer.setCountry(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of country must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Country")); + customer.setCountry("Ukraine"); customer.setPhone(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of phone must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Phone")); + customer.setPhone("+3892555554512"); customer.setState(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of state must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad state")); + customer.setState("Normal state"); customer.setZip(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of zip or postal code must be equal or less than 255"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Zip")); } @Test public void testUpdateCustomerFromDifferentTenant() throws Exception { Customer customer = new Customer(); customer.setTitle("My customer"); + + Mockito.reset(tbClusterService, auditLogService); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + + testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED); + doPost("/api/customer", savedCustomer, Customer.class); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/customer", savedCustomer, Customer.class, status().isForbidden()); + + testNotifyEntityNever(savedCustomer.getId(), savedCustomer); + deleteDifferentTenant(); login(tenantAdmin.getName(), "testPassword1"); + + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); + + testNotifyEntityDeleteOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedCustomer.getId().getId().toString()); } @Test public void testFindCustomerById() throws Exception { Customer customer = new Customer(); customer.setTitle("My customer"); + + Mockito.reset(tbClusterService, auditLogService); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), + tenantAdmin.getEmail(), ActionType.ADDED); + Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); Assert.assertNotNull(foundCustomer); Assert.assertEquals(savedCustomer, foundCustomer); doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); + + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedCustomer.getId().getId().toString()); } @Test @@ -159,36 +255,19 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest customer.setTitle("My customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedCustomer.getId().getId().toString()); + doGet("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isNotFound()); } - @Test - public void testSaveCustomerWithEmptyTitle() throws Exception { - Customer customer = new Customer(); - doPost("/api/customer", customer) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Customer title should be specified"))); - } - - @Test - public void testSaveCustomerWithInvalidEmail() throws Exception { - Customer customer = new Customer(); - customer.setTitle("My customer"); - customer.setEmail("invalid@mail"); - doPost("/api/customer", customer) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Invalid email address format 'invalid@mail'"))); - -// loginSysAdmin(); -// -// doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) -// .andExpect(status().isOk()); - } - @Test public void testFindCustomers() throws Exception { TenantId tenantId = savedTenant.getId(); From 6ebd4120c13311ded31671540df8da63247db959 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Tue, 14 Jun 2022 10:21:01 +0300 Subject: [PATCH 209/262] UI: Remove padding --- .../components/dashboard-page/add-widget-dialog.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/add-widget-dialog.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/add-widget-dialog.component.html index 99f5e6ebfd..9fa86d8f61 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/add-widget-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/add-widget-dialog.component.html @@ -28,7 +28,7 @@ -
+
Date: Tue, 14 Jun 2022 11:18:24 +0300 Subject: [PATCH 210/262] UI: Fixed vulnerabilities in tb-js-executor --- msa/js-executor/package.json | 35 +- msa/js-executor/pom.xml | 4 +- msa/js-executor/queue/serviceBusTemplate.js | 53 +- msa/js-executor/yarn.lock | 2317 +++++++++---------- 4 files changed, 1101 insertions(+), 1308 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 6cbfc1261d..d1df447072 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -12,20 +12,19 @@ "start-prod": "NODE_ENV=production nodemon server.js" }, "dependencies": { - "@azure/service-bus": "^1.1.9", - "@google-cloud/pubsub": "^2.5.0", - "amqplib": "^0.6.0", - "aws-sdk": "^2.741.0", - "azure-sb": "^0.11.1", - "config": "^3.3.1", - "express": "^4.17.1", - "js-yaml": "^3.14.0", - "kafkajs": "^1.15.0", - "long": "^4.0.0", + "@azure/service-bus": "^7.5.1", + "@google-cloud/pubsub": "^3.0.1", + "amqplib": "^0.10.0", + "aws-sdk": "^2.1152.0", + "config": "^3.3.7", + "express": "^4.18.1", + "js-yaml": "^4.1.0", + "kafkajs": "^2.0.2", + "long": "^5.2.0", "uuid-parse": "^1.1.0", "uuid-random": "^1.3.2", - "winston": "^3.3.3", - "winston-daily-rotate-file": "^4.5.0" + "winston": "^3.7.2", + "winston-daily-rotate-file": "^4.7.1" }, "nyc": { "exclude": [ @@ -36,13 +35,19 @@ ] }, "devDependencies": { - "fs-extra": "^10.0.0", - "nodemon": "^2.0.12", - "pkg": "^5.3.1" + "fs-extra": "^10.1.0", + "nodemon": "^2.0.16", + "pkg": "^5.7.0" }, "pkg": { "assets": [ "node_modules/config/**/*.*" ] + }, + "resolutions": { + "ansi-regex": "^5.0.1", + "color-string": "^1.5.5", + "minimist": "^1.2.6", + "node-fetch": "^2.6.7" } } diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index 5532a70b78..a910e4759f 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -71,8 +71,8 @@ install-node-and-yarn - v12.16.1 - v1.22.4 + v16.13.1 + v1.22.17 diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js index 9016640c67..e8efae1635 100644 --- a/msa/js-executor/queue/serviceBusTemplate.js +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -18,8 +18,7 @@ const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), logger = require('../config/logger')._logger('serviceBusTemplate'); -const {ServiceBusClient, ReceiveMode} = require("@azure/service-bus"); -const azure = require('azure-sb'); +const {ServiceBusClient, ServiceBusAdministrationClient} = require("@azure/service-bus"); const requestTopic = config.get('request_topic'); const namespaceName = config.get('service_bus.namespace_name'); @@ -28,7 +27,6 @@ const sasKey = config.get('service_bus.sas_key'); const queueProperties = config.get('service_bus.queue_properties'); let sbClient; -let receiverClient; let receiver; let serviceBusService; @@ -61,11 +59,10 @@ function ServiceBusProducer() { } function CustomSender(topic) { - this.queueClient = sbClient.createQueueClient(topic); - this.sender = this.queueClient.createSender(); + this.sender = sbClient.createSender(topic); this.send = async (message) => { - return this.sender.send(message); + return this.sender.sendMessages(message); } } @@ -74,8 +71,8 @@ function CustomSender(topic) { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`; - sbClient = ServiceBusClient.createFromConnectionString(connectionString); - serviceBusService = azure.createServiceBusService(connectionString); + sbClient = new ServiceBusClient(connectionString) + serviceBusService = new ServiceBusAdministrationClient(connectionString); parseQueueProperties(); @@ -84,9 +81,9 @@ function CustomSender(topic) { if (err) { reject(err); } else { - data.forEach(queue => { - queues.push(queue.QueueName); - }); + for (const queue of data) { + queues.push(queue.name); + } resolve(); } }); @@ -97,8 +94,7 @@ function CustomSender(topic) { queues.push(requestTopic); } - receiverClient = sbClient.createQueueClient(requestTopic); - receiver = receiverClient.createReceiver(ReceiveMode.peekLock); + receiver = sbClient.createReceiver(requestTopic, {receiveMode: 'peekLock'}); const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer()); @@ -111,18 +107,18 @@ function CustomSender(topic) { const errorHandler = (error) => { logger.error('Failed to receive message from queue.', error); }; - receiver.registerMessageHandler(messageHandler, errorHandler); + receiver.subscribe({processMessage: messageHandler, processError: errorHandler}) } catch (e) { logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); logger.error(e.stack); - exit(-1); + await exit(-1); } })(); async function createQueueIfNotExist(topic) { return new Promise((resolve, reject) => { - serviceBusService.createQueueIfNotExists(topic, queueOptions, (err) => { - if (err) { + serviceBusService.createQueue(topic, queueOptions, (err) => { + if (err && err.code !== "MessageEntityAlreadyExistsError") { reject(err); } else { resolve(); @@ -139,10 +135,10 @@ function parseQueueProperties() { properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); }); queueOptions = { - DuplicateDetection: 'false', - MaxSizeInMegabytes: properties['maxSizeInMb'], - DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, - LockDuration: `PT${properties['lockDurationInSec']}S` + requiresDuplicateDetection: false, + maxSizeInMegabytes: properties['maxSizeInMb'], + defaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, + lockDuration: `PT${properties['lockDurationInSec']}S` }; } @@ -161,24 +157,11 @@ async function exit(status) { } } - if (receiverClient) { - try { - await receiverClient.close(); - } catch (e) { - - } - } - senderMap.forEach((k, v) => { try { v.sender.close(); } catch (e) { - } - try { - v.queueClient.close(); - } catch (e) { - } }); @@ -191,4 +174,4 @@ async function exit(status) { } logger.info('Azure Service Bus resources stopped.') process.exit(status); -} \ No newline at end of file +} diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 0c46da466a..f93ad99803 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -9,75 +9,71 @@ dependencies: tslib "^1.9.3" -"@azure/amqp-common@1.0.0-preview.16": - version "1.0.0-preview.16" - resolved "https://registry.yarnpkg.com/@azure/amqp-common/-/amqp-common-1.0.0-preview.16.tgz#3b7a9d39f7503347530fe9afddf92fa7bfd1ec5e" - integrity sha512-rMTBh54lV6/SBujXfPX0OqN+UNpG1zR4EYY+JP66QM0D2jzNdxLiv6gart5Ersy4Tmd89MRlgikaj5+t5NsDgA== +"@azure/core-amqp@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@azure/core-amqp/-/core-amqp-3.1.0.tgz#f8e210cfbabbb5d163649eedeb80c0b69045d358" + integrity sha512-TyI0WFNrVb0EkRg36UwdcqR/7n9YpcEw64O4xVrgzMAlXIciVZpabl05C/Q0iUvLkItmez2XSVDduncjr02oGw== dependencies: - "@types/async-lock" "^1.1.0" - "@types/is-buffer" "^2.0.0" - async-lock "^1.1.3" - buffer "^5.2.1" - debug "^3.1.0" + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/logger" "^1.0.0" + buffer "^6.0.0" events "^3.0.0" - is-buffer "^2.0.3" - jssha "^2.3.1" + jssha "^3.1.0" process "^0.11.10" - rhea "^1.0.18" - rhea-promise "^0.1.15" - stream-browserify "^2.0.2" - tslib "^1.9.3" + rhea "^2.0.3" + rhea-promise "^2.1.0" + tslib "^2.2.0" url "^0.11.0" - util "^0.11.1" + util "^0.12.1" -"@azure/core-auth@^1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.1.3.tgz#94e7bbc207010e7a2fdba61565443e4e1cf1e131" - integrity sha512-A4xigW0YZZpkj1zK7dKuzbBpGwnhEcRk6WWuIshdHC32raR3EQ1j6VA9XZqE+RFsUgH6OAmIK5BWIz+mZjnd6Q== +"@azure/core-asynciterator-polyfill@^1.0.0": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@azure/core-asynciterator-polyfill/-/core-asynciterator-polyfill-1.0.2.tgz#0dd3849fb8d97f062a39db0e5cadc9ffaf861fec" + integrity sha512-3rkP4LnnlWawl0LZptJOdXNrT/fHp2eQMadoasa6afspXdpGrtPZuAQc2PD0cpgyuoXtUWyC3tv7xfntjGS5Dw== + +"@azure/core-auth@^1.3.0": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.3.2.tgz#6a2c248576c26df365f6c7881ca04b7f6d08e3d0" + integrity sha512-7CU6DmCHIZp5ZPiZ9r3J17lTKMmYsm/zGvNkjArQwPkrLlZ1TZ+EUYfGgh2X31OLMVAQCTJZW4cXHJi02EbJnA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-tracing" "1.0.0-preview.8" - "@opentelemetry/api" "^0.6.1" - tslib "^2.0.0" + tslib "^2.2.0" -"@azure/core-http@^1.0.0": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-1.1.6.tgz#9d76bc569c9907e3224bd09c09b4ac08bde9faf8" - integrity sha512-/C+qNzhwlLKt0F6SjaBEyY2pwZvwL2LviyS5PHlCh77qWuTF1sETmYAINM88BCN+kke+UlECK4YOQaAjJwyHvQ== +"@azure/core-http@^2.0.0": + version "2.2.5" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-2.2.5.tgz#e8cf8345d4a7f0ee7c8304a300c43d2df992ddbe" + integrity sha512-kctMqSQ6zfnlFpuYzfUKadeTyOQYbIQ+3Rj7dzVC3Dk1dOnHroTwR9hLYKX8/n85iJpkyaksaXpuh5L7GJRYuQ== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.1.3" - "@azure/core-tracing" "1.0.0-preview.9" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" - "@opentelemetry/api" "^0.10.2" "@types/node-fetch" "^2.5.0" - "@types/tunnel" "^0.0.1" - form-data "^3.0.0" - node-fetch "^2.6.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" process "^0.11.10" tough-cookie "^4.0.0" - tslib "^2.0.0" + tslib "^2.2.0" tunnel "^0.0.6" - uuid "^8.1.0" + uuid "^8.3.0" xml2js "^0.4.19" -"@azure/core-tracing@1.0.0-preview.8": - version "1.0.0-preview.8" - resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.8.tgz#1e0ff857e855edb774ffd33476003c27b5bb2705" - integrity sha512-ZKUpCd7Dlyfn7bdc+/zC/sf0aRIaNQMDuSj2RhYRFe3p70hVAnYGp3TX4cnG2yoEALp/LTj/XnZGQ8Xzf6Ja/Q== +"@azure/core-paging@^1.1.1": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.3.0.tgz#ebf6520a931be7d5ff1cf58bc4fe6c14d6a3080d" + integrity sha512-H6Tg9eBm0brHqLy0OSAGzxIh1t4UL8eZVrSUMJ60Ra9cwq2pOskFqVpz2pYoHDsBY1jZ4V/P8LRGb5D5pmC6rg== dependencies: - "@opencensus/web-types" "0.0.7" - "@opentelemetry/api" "^0.6.1" - tslib "^1.10.0" + tslib "^2.2.0" -"@azure/core-tracing@1.0.0-preview.9": - version "1.0.0-preview.9" - resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.9.tgz#84f3b85572013f9d9b85e1e5d89787aa180787eb" - integrity sha512-zczolCLJ5QG42AEPQ+Qg9SRYNUyB+yZ5dzof4YEc+dyWczO9G2sBqbAjLB7IqrsdHN2apkiB2oXeDKCsq48jug== +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== dependencies: - "@opencensus/web-types" "0.0.7" - "@opentelemetry/api" "^0.10.2" - tslib "^2.0.0" + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" "@azure/logger@^1.0.0": version "1.0.0" @@ -86,44 +82,52 @@ dependencies: tslib "^1.9.3" -"@azure/service-bus@^1.1.9": - version "1.1.9" - resolved "https://registry.yarnpkg.com/@azure/service-bus/-/service-bus-1.1.9.tgz#856cec37d0fbd8069fd9b717afa3aa9fdb48c95b" - integrity sha512-7i+7h1gh8vUeeaHddEVg+x7lXSVqhk/f2z4x4ErZByrE0QsyibFVPpBqS3q1RIjpDgmKPxcj7Wzu7O8vJLSaRw== +"@azure/service-bus@^7.5.1": + version "7.5.1" + resolved "https://registry.yarnpkg.com/@azure/service-bus/-/service-bus-7.5.1.tgz#5c006f29cf42779c20b87431e4482150b83be736" + integrity sha512-fIbI5aJDzN2HctcS+3i7rXY3L5jWG7/SuXOq0cMpwy+aH5aHOZVoxX8eeSnUnr2jUpUflU1wJ9yiYZ/sXqZvqw== dependencies: - "@azure/amqp-common" "1.0.0-preview.16" - "@azure/core-http" "^1.0.0" - "@opentelemetry/types" "^0.2.0" + "@azure/abort-controller" "^1.0.0" + "@azure/core-amqp" "^3.1.0" + "@azure/core-asynciterator-polyfill" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-http" "^2.0.0" + "@azure/core-paging" "^1.1.1" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/logger" "^1.0.0" "@types/is-buffer" "^2.0.0" - "@types/long" "^4.0.0" - buffer "^5.2.1" - debug "^4.1.1" + "@types/long" "^4.0.1" + buffer "^6.0.0" is-buffer "^2.0.3" + jssha "^3.1.0" long "^4.0.0" process "^0.11.10" - rhea "^1.0.23" - rhea-promise "^0.1.15" - tslib "^1.10.0" - -"@babel/helper-validator-identifier@^7.12.11": - version "7.14.9" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.9.tgz#6654d171b2024f6d8ee151bf2509699919131d48" - integrity sha512-pQYxPY0UP6IHISRitNe8bsijHex4TWZXi2HwKVsjPiltzlhse2znVcm9Ace510VT1kxIHjGJCZZQBX2gJDbo0g== - -"@babel/parser@7.13.13": - version "7.13.13" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.13.tgz#42f03862f4aed50461e543270916b47dd501f0df" - integrity sha512-OhsyMrqygfk5v8HmWwOzlYjJrtLaFhF34MrfG/Z73DgYCI6ojNUTUp2TYbtnjo8PegeJp12eamsNettCQjKjVw== - -"@babel/types@7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.12.tgz#edbf99208ef48852acdff1c8a681a1e4ade580cd" - integrity sha512-K4nY2xFN4QMvQwkQ+zmBDp6ANMbVNw6BbxWmYA4qNjhR9W+Lj/8ky5MEY2Me5r+B2c6/v6F53oMndG+f9s3IiA== - dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - lodash "^4.17.19" + rhea-promise "^2.1.0" + tslib "^2.2.0" + +"@babel/helper-validator-identifier@^7.16.7": + version "7.16.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" + integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== + +"@babel/parser@7.17.10": + version "7.17.10" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.10.tgz#873b16db82a8909e0fbd7f115772f4b739f6ce78" + integrity sha512-n2Q6i+fnJqzOaq2VkdXxy2TCPCWQZHiCo0XqmrCvDWcZQKRyZzYi4Z0yxlBuN0w+r2ZHmre+Q087DSrw3pbJDQ== + +"@babel/types@7.17.10": + version "7.17.10" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.17.10.tgz#d35d7b4467e439fcf06d195f8100e0fea7fc82c4" + integrity sha512-9O26jG0mBYfGkUYCYZRnBwbVLd1UZOICEr2Em6InB6jVfsAv1GKgwXHmrSg+WFWDmeKTA6vyTZiN8tCSM5Oo3A== + dependencies: + "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@colors/colors@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" + integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + "@dabh/diagnostics@^2.0.2": version "2.0.2" resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31" @@ -133,10 +137,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@google-cloud/paginator@^3.0.0": - version "3.0.4" - resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-3.0.4.tgz#4106cadd8153d157c8afd16d66dfb89a38ba8748" - integrity sha512-fKI+jYQdV1F9jtG6tSRro3ilNSeBWVmTzxc8Z0kiPRXcj8eshh9fiF8TtxfDefyUKgTdWgHpzGBwLbZ/OGikJg== +"@google-cloud/paginator@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-4.0.0.tgz#9c3e01544717aecb9a922b4269ff298f30a0f1bb" + integrity sha512-wNmCZl+2G2DmgT/VlF+AROf80SoaC/CwS8trwmjNaq26VRNK8yPbU5F/Vy+R9oDAGKWQU2k8+Op5H4kFJVXFaQ== dependencies: arrify "^2.0.0" extend "^3.0.2" @@ -156,55 +160,45 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.2.tgz#81d654b4cb227c65c7ad2f9a7715262febd409ed" integrity sha512-EvuabjzzZ9E2+OaYf+7P9OAiiwbTxKYL0oGLnREQd+Su2NTQBpomkdlkBowFvyWsaV0d1sSGxrKpSNcrhPqbxg== -"@google-cloud/pubsub@^2.5.0": - version "2.5.0" - resolved "https://registry.yarnpkg.com/@google-cloud/pubsub/-/pubsub-2.5.0.tgz#6c696d9b448f2e1689be9a37ef0362ed173731fd" - integrity sha512-7bbbQqa+LSTopVjt20EZ8maO6rEpbO7v8EvDImHMsbRS30HJ5+kClbaQTRvhNzhc1qy221A1GbHPHMCQ/U5E3Q== +"@google-cloud/pubsub@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@google-cloud/pubsub/-/pubsub-3.0.1.tgz#3a6bb9649a1b4309d82d8670a9c01375e622692f" + integrity sha512-dznNbRd/Y8J0C0xvdvCPi3B1msK/dj/Nya+NQZ2doUOLT6eoa261tBwk9umOQs5L5GKcdlqQKbBjrNjDYVbzQA== dependencies: - "@google-cloud/paginator" "^3.0.0" + "@google-cloud/paginator" "^4.0.0" "@google-cloud/precise-date" "^2.0.0" "@google-cloud/projectify" "^2.0.0" "@google-cloud/promisify" "^2.0.0" - "@opentelemetry/api" "^0.10.0" - "@opentelemetry/tracing" "^0.10.0" + "@opentelemetry/api" "^1.0.0" + "@opentelemetry/semantic-conventions" "^1.0.0" "@types/duplexify" "^3.6.0" "@types/long" "^4.0.0" arrify "^2.0.0" extend "^3.0.2" - google-auth-library "^6.0.0" - google-gax "^2.7.0" + google-auth-library "^8.0.2" + google-gax "^3.0.1" is-stream-ended "^0.1.4" lodash.snakecase "^4.1.1" p-defer "^3.0.0" -"@grpc/grpc-js@~1.1.1": - version "1.1.5" - resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.1.5.tgz#2d0b261cd54a529f6b78ac0de9d6fd91a9a3129c" - integrity sha512-2huf5z85TdZI4nLmJQ9Zdfd+6vmIyBDs7B4L71bTaHKA9pRsGKAH24XaktMk/xneKJIqAgeIZtg1cyivVZtvrg== +"@grpc/grpc-js@~1.6.0": + version "1.6.7" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.6.7.tgz#4c4fa998ff719fe859ac19fe977fdef097bb99aa" + integrity sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw== dependencies: - "@grpc/proto-loader" "^0.6.0-pre14" - "@types/node" "^12.12.47" - google-auth-library "^6.0.0" - semver "^6.2.0" - -"@grpc/proto-loader@^0.5.1": - version "0.5.5" - resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.5.5.tgz#6725e7a1827bdf8e92e29fbf4e9ef0203c0906a9" - integrity sha512-WwN9jVNdHRQoOBo9FDH7qU+mgfjPc8GygPYms3M+y3fbQLfnCe/Kv/E01t7JRgnrsOHH8euvSbed3mIalXhwqQ== - dependencies: - lodash.camelcase "^4.3.0" - protobufjs "^6.8.6" + "@grpc/proto-loader" "^0.6.4" + "@types/node" ">=12.12.47" -"@grpc/proto-loader@^0.6.0-pre14": - version "0.6.0-pre9" - resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.6.0-pre9.tgz#0c6fe42f6c5ef9ce1b3cef7be64d5b09d6fe4d6d" - integrity sha512-oM+LjpEjNzW5pNJjt4/hq1HYayNeQT+eGrOPABJnYHv7TyNPDNzkQ76rDYZF86X5swJOa4EujEMzQ9iiTdPgww== +"@grpc/proto-loader@^0.6.12", "@grpc/proto-loader@^0.6.4": + version "0.6.13" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.6.13.tgz#008f989b72a40c60c96cd4088522f09b05ac66bc" + integrity sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g== dependencies: "@types/long" "^4.0.1" lodash.camelcase "^4.3.0" long "^4.0.0" - protobufjs "^6.9.0" - yargs "^15.3.1" + protobufjs "^6.11.3" + yargs "^16.2.0" "@nodelib/fs.scandir@2.1.3": version "2.1.3" @@ -227,67 +221,15 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@opencensus/web-types@0.0.7": - version "0.0.7" - resolved "https://registry.yarnpkg.com/@opencensus/web-types/-/web-types-0.0.7.tgz#4426de1fe5aa8f624db395d2152b902874f0570a" - integrity sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g== - -"@opentelemetry/api@^0.10.0", "@opentelemetry/api@^0.10.2": - version "0.10.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-0.10.2.tgz#9647b881f3e1654089ff7ea59d587b2d35060654" - integrity sha512-GtpMGd6vkzDMYcpu2t9LlhEgMy/SzBwRnz48EejlRArYqZzqSzAsKmegUK7zHgl+EOIaK9mKHhnRaQu3qw20cA== - dependencies: - "@opentelemetry/context-base" "^0.10.2" - -"@opentelemetry/api@^0.6.1": - version "0.6.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-0.6.1.tgz#a00b504801f408230b9ad719716fe91ad888c642" - integrity sha512-wpufGZa7tTxw7eAsjXJtiyIQ42IWQdX9iUQp7ACJcKo1hCtuhLU+K2Nv1U6oRwT1oAlZTE6m4CgWKZBhOiau3Q== - dependencies: - "@opentelemetry/context-base" "^0.6.1" - -"@opentelemetry/context-base@^0.10.2": - version "0.10.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/context-base/-/context-base-0.10.2.tgz#55bea904b2b91aa8a8675df9eaba5961bddb1def" - integrity sha512-hZNKjKOYsckoOEgBziGMnBcX0M7EtstnCmwz5jZUOUYwlZ+/xxX6z3jPu1XVO2Jivk0eLfuP9GP+vFD49CMetw== +"@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.0.1": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.1.0.tgz#563539048255bbe1a5f4f586a4a10a1bb737f44a" + integrity sha512-hf+3bwuBwtXsugA2ULBc95qxrOqP2pOekLz34BJhcAKawt94vfeNyUKpYc0lZQ/3sCP6LqRa7UAdHA7i5UODzQ== -"@opentelemetry/context-base@^0.6.1": - version "0.6.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/context-base/-/context-base-0.6.1.tgz#b260e454ee4f9635ea024fc83be225e397f15363" - integrity sha512-5bHhlTBBq82ti3qPT15TRxkYTFPPQWbnkkQkmHPtqiS1XcTB69cEKd3Jm7Cfi/vkPoyxapmePE9tyA7EzLt8SQ== - -"@opentelemetry/core@^0.10.2": - version "0.10.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-0.10.2.tgz#86b9e94bbcaf8e07bb86e8205aa1d53af854e7de" - integrity sha512-DhkiTp5eje2zTGd+HAIKWpGE6IR6lq7tUpYt4nnkhOi6Hq9WQAANVDCWEZEbYOw57LkdXbE50FZ/kMvHDm450Q== - dependencies: - "@opentelemetry/api" "^0.10.2" - "@opentelemetry/context-base" "^0.10.2" - semver "^7.1.3" - -"@opentelemetry/resources@^0.10.2": - version "0.10.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-0.10.2.tgz#6e291d525450359c615aac013fd977047f2c26d7" - integrity sha512-5JGC2TPSAIHth615IURt+sSsTljY43zTfJD0JE9PHC6ipZPiQ0dpQDZOrLn8NAMfOHY1jeWwpIuLASjqbXUfuw== - dependencies: - "@opentelemetry/api" "^0.10.2" - "@opentelemetry/core" "^0.10.2" - gcp-metadata "^3.5.0" - -"@opentelemetry/tracing@^0.10.0": - version "0.10.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/tracing/-/tracing-0.10.2.tgz#384779a6e1be988200cc316a97030d95bd8f2129" - integrity sha512-mNAhARn4dEdOjTa9OdysjI4fRHMbvr4YSbPuH7jhkyPzgoa+DnvnbY3GGpEay6kpuYJsrW8Ef9OIKAV/GndhbQ== - dependencies: - "@opentelemetry/api" "^0.10.2" - "@opentelemetry/context-base" "^0.10.2" - "@opentelemetry/core" "^0.10.2" - "@opentelemetry/resources" "^0.10.2" - -"@opentelemetry/types@^0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/types/-/types-0.2.0.tgz#2a0afd40fa7026e39ea56a454642bda72b172f80" - integrity sha512-GtwNB6BNDdsIPAYEdpp3JnOGO/3AJxjPvny53s3HERBdXSJTGQw8IRhiaTEX0b3w9P8+FwFZde4k+qkjn67aVw== +"@opentelemetry/semantic-conventions@^1.0.0": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.3.1.tgz#ba07b864a3c955f061aa30ea3ef7f4ae4449794a" + integrity sha512-wU5J8rUoo32oSef/rFpOT1HIjLjAv3qIDHkw1QIhODV3OpAVHi5oVzlouozg9obUmZKtbZ0qUe/m7FP0y0yBzA== "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" @@ -354,11 +296,6 @@ dependencies: defer-to-connect "^1.0.1" -"@types/async-lock@^1.1.0": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@types/async-lock/-/async-lock-1.1.2.tgz#cbc26a34b11b83b28f7783a843c393b443ef8bef" - integrity sha512-j9n4bb6RhgFIydBe0+kpjnBPYumDaDyU8zvbWykyVMkku+c2CSu31MZkLeaBfqIwU+XCxlDpYDfyMQRkM0AkeQ== - "@types/color-name@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" @@ -396,20 +333,15 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.6.1.tgz#fdf6f6c6c73d3d8eee9c98a9a0485bc524b048d7" integrity sha512-HnYlg/BRF8uC1FyKRFZwRaCPTPYKa+6I8QiUZFLredaGOou481cgFS4wKRFyKvQtX8xudqkSdBczJHIYSQYKrQ== -"@types/node@^12.12.47": - version "12.12.54" - resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.54.tgz#a4b58d8df3a4677b6c08bfbc94b7ad7a7a5f82d1" - integrity sha512-ge4xZ3vSBornVYlDnk7yZ0gK6ChHf/CHB7Gl1I0Jhah8DDnEQqBzgohYG4FX4p81TNirSETOiSyn+y1r9/IR6w== - -"@types/node@^13.7.0": - version "13.13.15" - resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.15.tgz#fe1cc3aa465a3ea6858b793fd380b66c39919766" - integrity sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw== +"@types/node@>=12.12.47", "@types/node@>=13.7.0": + version "17.0.42" + resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.42.tgz#d7e8f22700efc94d125103075c074396b5f41f9b" + integrity sha512-Q5BPGyGKcvQgAMbsr7qEGN/kIPN6zZecYYABeTDBizOsau+2NMdSVTar9UQw21A2+JyA2KRNDYaYrPB0Rpk2oQ== -"@types/tunnel@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.1.tgz#0d72774768b73df26f25df9184273a42da72b19c" - integrity sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== dependencies: "@types/node" "*" @@ -425,13 +357,13 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" -accepts@~1.3.7: - version "1.3.7" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" - integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== +accepts@~1.3.8: + version "1.3.8" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" + integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== dependencies: - mime-types "~2.1.24" - negotiator "0.6.2" + mime-types "~2.1.34" + negotiator "0.6.3" agent-base@6: version "6.0.1" @@ -440,27 +372,15 @@ agent-base@6: dependencies: debug "4" -ajv@^6.12.3: - version "6.12.4" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.4.tgz#0614facc4522127fa713445c6bfd3ebd376e2234" - integrity sha512-eienB2c9qVQs2KWexhkrdMLVDoIQCz5KSeLxwg9Lzk4DOfBtIK9PQwwufcsn1jjGuf9WZmqPMbGxOzfcuphJCQ== - dependencies: - fast-deep-equal "^3.1.1" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.4.1" - uri-js "^4.2.2" - -amqplib@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/amqplib/-/amqplib-0.6.0.tgz#87857c7c95d56d22438ced4cf1f7e5f0dc43b309" - integrity sha512-zXCh4jQ77TBZe1YtvZ1n7sUxnTjnNagpy8MVi2yc1ive239pS3iLwm4e4d5o4XZGx1BdTKQ/U0ZmaDU3c8MxYQ== +amqplib@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/amqplib/-/amqplib-0.10.0.tgz#766d696f8ceae097ee9eb73e6796999e5d40a1db" + integrity sha512-UueEnRGY6upiSvGsSYM22Woa1SeSukqYtqgYW4Gj8gHvbf5BRhhYRqf3kQ8aSUYYffTOZi6SeOVW2eOXt0hpPA== dependencies: bitsyntax "~0.1.0" - bluebird "^3.5.2" buffer-more-ints "~1.0.0" readable-stream "1.x >=1.1.9" - safe-buffer "~5.1.2" - url-parse "~1.4.3" + url-parse "~1.5.10" ansi-align@^3.0.0: version "3.0.0" @@ -469,25 +389,10 @@ ansi-align@^3.0.0: dependencies: string-width "^3.0.0" -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= - -ansi-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" - integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg= - -ansi-regex@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" - integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== - -ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== +ansi-regex@^2.0.0, ansi-regex@^3.0.0, ansi-regex@^4.1.0, ansi-regex@^5.0.0, ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.2.1" @@ -497,10 +402,10 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: "@types/color-name" "^1.1.1" color-convert "^2.0.1" -anymatch@~3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.1.tgz#c55ecf02185e2469259399310c173ce31233b142" - integrity sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg== +anymatch@~3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" + integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== dependencies: normalize-path "^3.0.0" picomatch "^2.0.4" @@ -518,12 +423,10 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" -argparse@^1.0.7: - version "1.0.10" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" - integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== - dependencies: - sprintf-js "~1.0.2" +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== array-flatten@1.1.1: version "1.1.1" @@ -540,27 +443,10 @@ arrify@^2.0.0: resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== -asn1@~0.2.3: - version "0.2.4" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" - integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== - dependencies: - safer-buffer "~2.1.0" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= - -async-lock@^1.1.3: - version "1.2.4" - resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.2.4.tgz#80d0d612383045dd0c30eb5aad08510c1397cb91" - integrity sha512-UBQJC2pbeyGutIfYmErGc9RaJYnpZ1FHaxuKwb0ahvGiiCkPUf3p67Io+YLPmmv3RHY+mF6JEtNW8FlHsraAaA== - -async@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== +async@^3.2.3: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== asynckit@^0.4.0: version "0.4.0" @@ -572,57 +458,26 @@ at-least-node@^1.0.0: resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== -aws-sdk@^2.741.0: - version "2.741.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.741.0.tgz#b832d838e5f1ef9cdb2b4901ade4555b1ca4c085" - integrity sha512-bpk5VvlBSvKu3Lg30AxX+PHk+0TD69K3tYe6D6VeEFl/3XzuZ5RKTGCIl96isSMyYc5bBMhLS9pC9glrxT7OTw== +available-typed-arrays@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" + integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== + +aws-sdk@^2.1152.0: + version "2.1152.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1152.0.tgz#73e4fb81b3a9c289234b5d6848bcdb854f169bdf" + integrity sha512-Lqwk0bDhm3vzpYb3AAM9VgGHeDpbB8+o7UJnP9R+CO23kJfi/XRpKihAcbyKDD/AUQ+O1LJaUVpvaJYLS9Am7w== dependencies: buffer "4.9.2" events "1.1.1" ieee754 "1.1.13" - jmespath "0.15.0" + jmespath "0.16.0" querystring "0.2.0" sax "1.2.1" url "0.10.3" - uuid "3.3.2" + uuid "8.0.0" xml2js "0.4.19" -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= - -aws4@^1.8.0: - version "1.10.1" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.10.1.tgz#e1e82e4f3e999e2cfd61b161280d16a111f86428" - integrity sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA== - -azure-common@^0.9.22: - version "0.9.25" - resolved "https://registry.yarnpkg.com/azure-common/-/azure-common-0.9.25.tgz#fc7cfb08c65398b3c90f9b9bc14a99e204319b9a" - integrity sha512-L7YO3DUQ0iwiaUyD9Wy6B66Y6HmCzMb9vxUqKklgzU+gFRRBKIMSVR4oZS6IkQfFCSm9eKwHuH2p3UdDPszd7g== - dependencies: - dateformat "1.0.2-1.2.3" - duplexer "~0.1.1" - envconf "~0.0.4" - request "^2.81.0" - through "~2.3.4" - tunnel "~0.0.2" - underscore "1.4.x" - validator "^9.4.1" - xml2js "^0.4.19" - xmlbuilder "15.1.1" - -azure-sb@^0.11.1: - version "0.11.1" - resolved "https://registry.yarnpkg.com/azure-sb/-/azure-sb-0.11.1.tgz#15cba11c1a3b4aca68c344460b99bceec14be11a" - integrity sha512-ZYgPeSDMD99i/Em+6wT78zvBkJ/dbh2ypb4DbqQ1Flaif5vWJFzC/iKxxcq/vq+THWoO3+UbqWa0JNXnW3zAvw== - dependencies: - azure-common "^0.9.22" - mpns "2.1.3" - underscore "^1.8.3" - wns "~0.5.3" - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -638,13 +493,6 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= - dependencies: - tweetnacl "^0.14.3" - bignumber.js@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.0.tgz#805880f84a329b5eac6e7cb6f8274b6d82bdf075" @@ -673,40 +521,37 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" -bluebird@^3.5.2: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - -body-parser@1.19.0: - version "1.19.0" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" - integrity sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw== +body-parser@1.20.0: + version "1.20.0" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.0.tgz#3de69bd89011c11573d7bfee6a64f11b6bd27cc5" + integrity sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg== dependencies: - bytes "3.1.0" + bytes "3.1.2" content-type "~1.0.4" debug "2.6.9" - depd "~1.1.2" - http-errors "1.7.2" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.7.0" - raw-body "2.4.0" - type-is "~1.6.17" + on-finished "2.4.1" + qs "6.10.3" + raw-body "2.5.1" + type-is "~1.6.18" + unpipe "1.0.0" -boxen@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" - integrity sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ== +boxen@^5.0.0: + version "5.1.2" + resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" + integrity sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ== dependencies: ansi-align "^3.0.0" - camelcase "^5.3.1" - chalk "^3.0.0" - cli-boxes "^2.2.0" - string-width "^4.1.0" - term-size "^2.1.0" - type-fest "^0.8.1" + camelcase "^6.2.0" + chalk "^4.1.0" + cli-boxes "^2.2.1" + string-width "^4.2.2" + type-fest "^0.20.2" widest-line "^3.1.0" + wrap-ansi "^7.0.0" brace-expansion@^1.1.7: version "1.1.11" @@ -716,7 +561,7 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -braces@^3.0.1, braces@~3.0.2: +braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -742,14 +587,6 @@ buffer@4.9.2: ieee754 "^1.1.4" isarray "^1.0.0" -buffer@^5.2.1: - version "5.6.0" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" - integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -758,10 +595,18 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -bytes@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" - integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== +buffer@^6.0.0: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + +bytes@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" + integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== cacheable-request@^6.0.0: version "6.1.0" @@ -776,25 +621,20 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" -camelcase@^5.0.0, camelcase@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" - integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== - -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= - -chalk@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" - integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== +call-bind@^1.0.0, call-bind@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +camelcase@^6.2.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -chalk@^4.1.0: +chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -802,20 +642,20 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chokidar@^3.2.2: - version "3.4.2" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.2.tgz#38dc8e658dec3809741eb3ef7bb0a47fe424232d" - integrity sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A== +chokidar@^3.5.2: + version "3.5.3" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" + integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== dependencies: - anymatch "~3.1.1" + anymatch "~3.1.2" braces "~3.0.2" - glob-parent "~5.1.0" + glob-parent "~5.1.2" is-binary-path "~2.1.0" is-glob "~4.0.1" normalize-path "~3.0.0" - readdirp "~3.4.0" + readdirp "~3.6.0" optionalDependencies: - fsevents "~2.1.2" + fsevents "~2.3.2" chownr@^1.1.1: version "1.1.4" @@ -827,19 +667,10 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== -cli-boxes@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.0.tgz#538ecae8f9c6ca508e3c3c95b453fe93cb4c168d" - integrity sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w== - -cliui@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1" - integrity sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ== - dependencies: - string-width "^4.2.0" - strip-ansi "^6.0.0" - wrap-ansi "^6.2.0" +cli-boxes@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" + integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== cliui@^7.0.2: version "7.0.4" @@ -886,10 +717,10 @@ color-name@^1.0.0, color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -color-string@^1.5.2: - version "1.5.3" - resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.3.tgz#c9bbc5f01b58b5492f3d6857459cb6590ce204cc" - integrity sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw== +color-string@^1.5.2, color-string@^1.5.5: + version "1.9.1" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" + integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== dependencies: color-name "^1.0.0" simple-swizzle "^0.2.2" @@ -902,11 +733,6 @@ color@3.0.x: color-convert "^1.9.1" color-string "^1.5.2" -colors@^1.2.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== - colorspace@1.1.x: version "1.1.2" resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5" @@ -915,7 +741,7 @@ colorspace@1.1.x: color "3.0.x" text-hex "1.0.x" -combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: +combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -927,10 +753,10 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= -config@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/config/-/config-3.3.1.tgz#b6a70e2908a43b98ed20be7e367edf0cc8ed5a19" - integrity sha512-+2/KaaaAzdwUBE3jgZON11L1ggLLhpf2FsGrfqYFHZW22ySGv/HqYIXrBwKKvn+XZh1UBUjHwAcrfsSkSygT+Q== +config@^3.3.7: + version "3.3.7" + resolved "https://registry.yarnpkg.com/config/-/config-3.3.7.tgz#4310410dc2bf4e0effdca21a12a4035860a24ee4" + integrity sha512-mX/n7GKDYZMqvvkY6e6oBY49W8wxdmQt+ho/5lhwFDXqQW9gI+Ahp8EKp8VAbISPnmf2+Bv5uZK7lKXZ6pf1aA== dependencies: json5 "^2.1.1" @@ -951,12 +777,12 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0: resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= -content-disposition@0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" - integrity sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g== +content-disposition@0.5.4: + version "0.5.4" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" + integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== dependencies: - safe-buffer "5.1.2" + safe-buffer "5.2.1" content-type@~1.0.4: version "1.0.4" @@ -968,12 +794,12 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= -cookie@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba" - integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg== +cookie@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" + integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= @@ -983,26 +809,14 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= - dependencies: - assert-plus "^1.0.0" - -dateformat@1.0.2-1.2.3: - version "1.0.2-1.2.3" - resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-1.0.2-1.2.3.tgz#b0220c02de98617433b72851cf47de3df2cdbee9" - integrity sha1-sCIMAt6YYXQztyhRz0fePfLNvuk= - -"debug@0.8.0 - 3.5.0", debug@^3.1.0, debug@^3.2.6: +"debug@0.8.0 - 3.5.0", debug@^3.1.0: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== dependencies: ms "^2.1.1" -debug@2.6.9, debug@^2.2.0, debug@~2.6.9: +debug@2.6.9, debug@~2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -1016,10 +830,12 @@ debug@4, debug@^4.1.1: dependencies: ms "^2.1.1" -decamelize@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" - integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" decompress-response@^3.3.0: version "3.3.0" @@ -1050,6 +866,14 @@ defer-to-connect@^1.0.1: resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== +define-properties@^1.1.3, define-properties@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" + integrity sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA== + dependencies: + has-property-descriptors "^1.0.0" + object-keys "^1.1.1" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -1060,15 +884,15 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= -depd@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" - integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= +depd@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-libc@^1.0.3: version "1.0.3" @@ -1094,29 +918,16 @@ duplexer3@^0.1.4: resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI= -duplexer@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" - integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== - -duplexify@^3.6.0: - version "3.7.1" - resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309" - integrity sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g== +duplexify@^4.0.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" + integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== dependencies: - end-of-stream "^1.0.0" - inherits "^2.0.1" - readable-stream "^2.0.0" + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" stream-shift "^1.0.0" -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" @@ -1149,17 +960,50 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= -end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: +end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== dependencies: once "^1.4.0" -envconf@~0.0.4: - version "0.0.4" - resolved "https://registry.yarnpkg.com/envconf/-/envconf-0.0.4.tgz#85675afba237c43f98de2d46adc0e532a4dcf48b" - integrity sha1-hWda+6I3xD+Y3i1GrcDlMqTc9Is= +es-abstract@^1.19.0, es-abstract@^1.19.5, es-abstract@^1.20.0: + version "1.20.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.1.tgz#027292cd6ef44bd12b1913b828116f54787d1814" + integrity sha512-WEm2oBhfoI2sImeM4OF2zE2V3BYdSF+KnSi9Sidz51fQHd7+JuF8Xgcj9/0o+OWeIeIS/MiuNnlruQrJf16GQA== + dependencies: + call-bind "^1.0.2" + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + function.prototype.name "^1.1.5" + get-intrinsic "^1.1.1" + get-symbol-description "^1.0.0" + has "^1.0.3" + has-property-descriptors "^1.0.0" + has-symbols "^1.0.3" + internal-slot "^1.0.3" + is-callable "^1.2.4" + is-negative-zero "^2.0.2" + is-regex "^1.1.4" + is-shared-array-buffer "^1.0.2" + is-string "^1.0.7" + is-weakref "^1.0.2" + object-inspect "^1.12.0" + object-keys "^1.1.1" + object.assign "^4.1.2" + regexp.prototype.flags "^1.4.3" + string.prototype.trimend "^1.0.5" + string.prototype.trimstart "^1.0.5" + unbox-primitive "^1.0.2" + +es-to-primitive@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== + dependencies: + is-callable "^1.1.4" + is-date-object "^1.0.1" + is-symbol "^1.0.2" escalade@^3.1.1: version "3.1.1" @@ -1188,7 +1032,7 @@ escodegen@^2.0.0: optionalDependencies: source-map "~0.6.1" -esprima@^4.0.0, esprima@^4.0.1: +esprima@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -1228,90 +1072,65 @@ expand-template@^2.0.3: resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== -express@^4.17.1: - version "4.17.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" - integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g== +express@^4.18.1: + version "4.18.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.18.1.tgz#7797de8b9c72c857b9cd0e14a5eea80666267caf" + integrity sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q== dependencies: - accepts "~1.3.7" + accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.19.0" - content-disposition "0.5.3" + body-parser "1.20.0" + content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.4.0" + cookie "0.5.0" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.2" + depd "2.0.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "~1.1.2" + finalhandler "1.2.0" fresh "0.5.2" + http-errors "2.0.0" merge-descriptors "1.0.1" methods "~1.1.2" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" path-to-regexp "0.1.7" - proxy-addr "~2.0.5" - qs "6.7.0" + proxy-addr "~2.0.7" + qs "6.10.3" range-parser "~1.2.1" - safe-buffer "5.1.2" - send "0.17.1" - serve-static "1.14.1" - setprototypeof "1.1.1" - statuses "~1.5.0" + safe-buffer "5.2.1" + send "0.18.0" + serve-static "1.15.0" + setprototypeof "1.2.0" + statuses "2.0.1" type-is "~1.6.18" utils-merge "1.0.1" vary "~1.1.2" -extend@^3.0.2, extend@~3.0.2: +extend@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= - -extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= - -fast-deep-equal@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" - integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== - -fast-glob@^3.1.1: - version "3.2.4" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3" - integrity sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ== +fast-glob@^3.2.9: + version "3.2.11" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9" + integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.0" + glob-parent "^5.1.2" merge2 "^1.3.0" - micromatch "^4.0.2" - picomatch "^2.2.1" - -fast-json-stable-stringify@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" - integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + micromatch "^4.0.4" fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fast-safe-stringify@^2.0.4: - version "2.0.7" - resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" - integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== - -fast-text-encoding@^1.0.0: +fast-text-encoding@^1.0.0, fast-text-encoding@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== @@ -1328,12 +1147,12 @@ fecha@^4.2.0: resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== -file-stream-rotator@^0.5.7: - version "0.5.7" - resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.5.7.tgz#868a2e5966f7640a17dd86eda0e4467c089f6286" - integrity sha512-VYb3HZ/GiAGUCrfeakO8Mp54YGswNUHvL7P09WQcXAJNSj3iQ5QraYSp3cIn1MUyw6uzfgN/EFOarCNa4JvUHQ== +file-stream-rotator@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz#007019e735b262bb6c6f0197e58e5c87cb96cec3" + integrity sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ== dependencies: - moment "^2.11.2" + moment "^2.29.1" fill-range@^7.0.1: version "7.0.1" @@ -1342,36 +1161,30 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== +finalhandler@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== dependencies: debug "2.6.9" encodeurl "~1.0.2" escape-html "~1.0.3" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" - statuses "~1.5.0" + statuses "2.0.1" unpipe "~1.0.0" -find-up@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" - integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== - dependencies: - locate-path "^5.0.0" - path-exists "^4.0.0" - fn.name@1.x.x: version "1.1.0" resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= +for-each@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" + integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== + dependencies: + is-callable "^1.1.3" form-data@^3.0.0: version "3.0.0" @@ -1382,13 +1195,13 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== dependencies: asynckit "^0.4.0" - combined-stream "^1.0.6" + combined-stream "^1.0.8" mime-types "^2.1.12" forwarded@0.2.0: @@ -1414,10 +1227,10 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.0.0.tgz#9ff61b655dde53fb34a82df84bb214ce802e17c1" - integrity sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ== +fs-extra@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== dependencies: graceful-fs "^4.2.0" jsonfile "^6.0.1" @@ -1433,16 +1246,31 @@ fs-extra@^9.1.0: jsonfile "^6.0.1" universalify "^2.0.0" -fsevents@~2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" - integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fsevents@~2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function.prototype.name@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" + integrity sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.0" + functions-have-names "^1.2.2" + +functions-have-names@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" + integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== + gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -1457,49 +1285,50 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" -gaxios@^2.1.0: - version "2.3.4" - resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-2.3.4.tgz#eea99353f341c270c5f3c29fc46b8ead56f0a173" - integrity sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA== +gaxios@^4.0.0: + version "4.3.3" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.3.tgz#d44bdefe52d34b6435cc41214fdb160b64abfc22" + integrity sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA== dependencies: abort-controller "^3.0.0" extend "^3.0.2" https-proxy-agent "^5.0.0" is-stream "^2.0.0" - node-fetch "^2.3.0" + node-fetch "^2.6.7" -gaxios@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-3.1.0.tgz#95f65f5a335f61aff602fe124cfdba8524f765fa" - integrity sha512-DDTn3KXVJJigtz+g0J3vhcfbDbKtAroSTxauWsdnP57sM5KZ3d2c/3D9RKFJ86s43hfw6WULg6TXYw/AYiBlpA== +gaxios@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-5.0.0.tgz#df11e5d0a45831dd39eb5fbbba0d6a6b09815e70" + integrity sha512-VD/yc5ln6XU8Ch1hyYY6kRMBE0Yc2np3fPyeJeYHhrPs1i8rgnsApPMWyrugkl7LLoSqpOJVBWlQIa87OAvt8Q== dependencies: abort-controller "^3.0.0" extend "^3.0.2" https-proxy-agent "^5.0.0" is-stream "^2.0.0" - node-fetch "^2.3.0" + node-fetch "^2.6.7" -gcp-metadata@^3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-3.5.0.tgz#6d28343f65a6bbf8449886a0c0e4a71c77577055" - integrity sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA== - dependencies: - gaxios "^2.1.0" - json-bigint "^0.3.0" - -gcp-metadata@^4.1.0: - version "4.1.4" - resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.1.4.tgz#3adadb9158c716c325849ee893741721a3c09e7e" - integrity sha512-5J/GIH0yWt/56R3dNaNWPGQ/zXsZOddYECfJaqxFWgrZ9HC2Kvc5vl9upOgUUHKzURjAVf2N+f6tEJiojqXUuA== +gcp-metadata@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-5.0.0.tgz#a00f999f60a4461401e7c515f8a3267cfb401ee7" + integrity sha512-gfwuX3yA3nNsHSWUL4KG90UulNiq922Ukj3wLTrcnX33BB7PwB1o0ubR8KVvXu9nJH+P5w1j2SQSNNqto+H0DA== dependencies: - gaxios "^3.0.0" + gaxios "^5.0.0" json-bigint "^1.0.0" -get-caller-file@^2.0.1, get-caller-file@^2.0.5: +get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz#336975123e05ad0b7ba41f152ee4aadbea6cf598" + integrity sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + get-stream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -1514,85 +1343,85 @@ get-stream@^5.1.0: dependencies: pump "^3.0.0" -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= +get-symbol-description@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6" + integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== dependencies: - assert-plus "^1.0.0" + call-bind "^1.0.2" + get-intrinsic "^1.1.1" github-from-package@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= -glob-parent@^5.1.0, glob-parent@~5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" - integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ== +glob-parent@^5.1.2, glob-parent@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" -global-dirs@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.0.1.tgz#acdf3bb6685bcd55cb35e8a052266569e9469201" - integrity sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A== +global-dirs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" + integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== dependencies: - ini "^1.3.5" + ini "2.0.0" -globby@^11.0.3: - version "11.0.4" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5" - integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg== +globby@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" + integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== dependencies: array-union "^2.1.0" dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" + fast-glob "^3.2.9" + ignore "^5.2.0" + merge2 "^1.4.1" slash "^3.0.0" -google-auth-library@^6.0.0: - version "6.0.6" - resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.0.6.tgz#5102e5c643baab45b4c16e9752cd56b8861f3a82" - integrity sha512-fWYdRdg55HSJoRq9k568jJA1lrhg9i2xgfhVIMJbskUmbDpJGHsbv9l41DGhCDXM21F9Kn4kUwdysgxSYBYJUw== +google-auth-library@^8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-8.0.2.tgz#5fa0f2d3795c3e4019d2bb315ade4454cc9c30b5" + integrity sha512-HoG+nWFAThLovKpvcbYzxgn+nBJPTfAwtq0GxPN821nOO+21+8oP7MoEHfd1sbDulUFFGfcjJr2CnJ4YssHcyg== dependencies: arrify "^2.0.0" base64-js "^1.3.0" ecdsa-sig-formatter "^1.0.11" fast-text-encoding "^1.0.0" - gaxios "^3.0.0" - gcp-metadata "^4.1.0" - gtoken "^5.0.0" + gaxios "^5.0.0" + gcp-metadata "^5.0.0" + gtoken "^5.3.2" jws "^4.0.0" lru-cache "^6.0.0" -google-gax@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/google-gax/-/google-gax-2.7.0.tgz#5aaff7bdbe32730f975f416dd4897c558c89ab84" - integrity sha512-0dBATy8mMVlfOBrT85Q+NzBpZ4OJZUMrPI9wJULpiIDq2w1zlN30Duor+fQUcMEjanYEc72G58M4iUVve0jfXw== +google-gax@^3.0.1: + version "3.1.0" + resolved "https://registry.yarnpkg.com/google-gax/-/google-gax-3.1.0.tgz#a6fea06bfd0157969ddc0a6d04c47980978ead02" + integrity sha512-OHKVHZtaYHwxrjiI4BH+IzRWH2Q75sZg+q0/RatxLicbeeCCIug99+NKQiE39B5rLnGENtcbUjmXiqJHbSrkSg== dependencies: - "@grpc/grpc-js" "~1.1.1" - "@grpc/proto-loader" "^0.5.1" + "@grpc/grpc-js" "~1.6.0" + "@grpc/proto-loader" "^0.6.12" "@types/long" "^4.0.0" abort-controller "^3.0.0" - duplexify "^3.6.0" - google-auth-library "^6.0.0" + duplexify "^4.0.0" + fast-text-encoding "^1.0.3" + google-auth-library "^8.0.2" is-stream-ended "^0.1.4" - lodash.at "^4.6.0" - lodash.has "^4.5.2" - node-fetch "^2.6.0" - protobufjs "^6.9.0" - retry-request "^4.0.0" - semver "^6.0.0" - walkdir "^0.4.0" + node-fetch "^2.6.1" + object-hash "^3.0.0" + proto3-json-serializer "^1.0.0" + protobufjs "6.11.3" + retry-request "^5.0.0" -google-p12-pem@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.0.2.tgz#12d443994b6f4cd8c9e4ac479f2f18d4694cbdb8" - integrity sha512-tbjzndQvSIHGBLzHnhDs3cL4RBjLbLXc2pYvGH+imGVu5b4RMAttUTdnmW2UH0t11QeBTXZ7wlXPS7hrypO/tg== +google-p12-pem@^3.1.3: + version "3.1.4" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.4.tgz#123f7b40da204de4ed1fbf2fd5be12c047fc8b3b" + integrity sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg== dependencies: - node-forge "^0.9.0" + node-forge "^1.3.1" got@^9.6.0: version "9.6.0" @@ -1616,28 +1445,19 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== -gtoken@^5.0.0: - version "5.0.3" - resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.0.3.tgz#b76ef8e9a2fed6fef165e47f7d05b60c498e4d05" - integrity sha512-Nyd1wZCMRc2dj/mAD0LlfQLcAO06uKdpKJXvK85SGrF5+5+Bpfil9u/2aw35ltvEHjvl0h5FMKN5knEU+9JrOg== +gtoken@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.2.tgz#deb7dc876abe002178e0515e383382ea9446d58f" + integrity sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ== dependencies: - gaxios "^3.0.0" - google-p12-pem "^3.0.0" + gaxios "^4.0.0" + google-p12-pem "^3.1.3" jws "^4.0.0" - mime "^2.2.0" -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= - -har-validator@~5.1.3: - version "5.1.5" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" - integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== - dependencies: - ajv "^6.12.3" - har-schema "^2.0.0" +has-bigints@^1.0.1, has-bigints@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" + integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== has-flag@^3.0.0: version "3.0.0" @@ -1649,6 +1469,25 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-property-descriptors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz#610708600606d36961ed04c196193b6a607fa861" + integrity sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ== + dependencies: + get-intrinsic "^1.1.1" + +has-symbols@^1.0.1, has-symbols@^1.0.2, has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +has-tostringtag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" + integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== + dependencies: + has-symbols "^1.0.2" + has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" @@ -1671,36 +1510,16 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== -http-errors@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" - integrity sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - -http-errors@~1.7.2: - version "1.7.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" - integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== +http-errors@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: - depd "~1.1.2" + depd "2.0.0" inherits "2.0.4" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" + setprototypeof "1.2.0" + statuses "2.0.1" + toidentifier "1.0.1" https-proxy-agent@^5.0.0: version "5.0.0" @@ -1722,7 +1541,7 @@ ieee754@1.1.13, ieee754@^1.1.4: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== -ieee754@^1.1.13: +ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -1732,10 +1551,10 @@ ignore-by-default@^1.0.1: resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" integrity sha1-SMptcvbGo68Aqa1K5odr44ieKwk= -ignore@^5.1.4: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== +ignore@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" + integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== import-lazy@^2.1.0: version "2.1.0" @@ -1747,21 +1566,30 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -inherits@2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= - inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@^1.3.5, ini@~1.3.0: +ini@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + +ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== +internal-slot@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c" + integrity sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA== + dependencies: + get-intrinsic "^1.1.0" + has "^1.0.3" + side-channel "^1.0.4" + into-stream@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-6.0.0.tgz#4bfc1244c0128224e18b8870e85b2de8e66c6702" @@ -1775,11 +1603,26 @@ ipaddr.js@1.9.1: resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== +is-arguments@^1.0.4: + version "1.1.1" + resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b" + integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ== +is-bigint@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3" + integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg== + dependencies: + has-bigints "^1.0.1" + is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" @@ -1787,11 +1630,24 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" +is-boolean-object@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719" + integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + is-buffer@^2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.4.tgz#3e572f23c8411a5cfd9557c849e3665e0b290623" integrity sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A== +is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.4.tgz#47301d58dd0259407865547853df6d61fe471945" + integrity sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w== + is-ci@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" @@ -1799,13 +1655,20 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.2.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.6.0.tgz#d7553b2526fe59b92ba3e40c8df757ec8a709e19" - integrity sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ== +is-core-module@2.9.0, is-core-module@^2.8.1: + version "2.9.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69" + integrity sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A== dependencies: has "^1.0.3" +is-date-object@^1.0.1: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f" + integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== + dependencies: + has-tostringtag "^1.0.0" + is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" @@ -1828,6 +1691,13 @@ is-fullwidth-code-point@^3.0.0: resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== +is-generator-function@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.0.10.tgz#f1558baf1ac17e0deea7c0415c438351ff2b3c72" + integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A== + dependencies: + has-tostringtag "^1.0.0" + is-glob@^4.0.1, is-glob@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" @@ -1835,18 +1705,30 @@ is-glob@^4.0.1, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-installed-globally@^0.3.1: - version "0.3.2" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.3.2.tgz#fd3efa79ee670d1187233182d5b0a1dd00313141" - integrity sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g== +is-installed-globally@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" + integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== dependencies: - global-dirs "^2.0.1" - is-path-inside "^3.0.1" + global-dirs "^3.0.0" + is-path-inside "^3.0.2" -is-npm@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d" - integrity sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig== +is-negative-zero@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" + integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA== + +is-npm@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-5.0.0.tgz#43e8d65cc56e1b67f8d47262cf667099193f45a8" + integrity sha512-WW/rQLOazUq+ST/bCAVBp/2oMERWLsR7OrKyt052dNDk4DHcDE0/7QSXITlmi+VBcV13DfIbysG3tZJm5RfdBA== + +is-number-object@^1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc" + integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ== + dependencies: + has-tostringtag "^1.0.0" is-number@^7.0.0: version "7.0.0" @@ -1858,10 +1740,25 @@ is-obj@^2.0.0: resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== -is-path-inside@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" - integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== +is-path-inside@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== + +is-regex@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958" + integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + +is-shared-array-buffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" + integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== + dependencies: + call-bind "^1.0.2" is-stream-ended@^0.1.4: version "0.1.4" @@ -1873,11 +1770,43 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== -is-typedarray@^1.0.0, is-typedarray@~1.0.0: +is-string@^1.0.5, is-string@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" + integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== + dependencies: + has-tostringtag "^1.0.0" + +is-symbol@^1.0.2, is-symbol@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" + integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== + dependencies: + has-symbols "^1.0.2" + +is-typed-array@^1.1.3, is-typed-array@^1.1.9: + version "1.1.9" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.9.tgz#246d77d2871e7d9f5aeb1d54b9f52c71329ece67" + integrity sha512-kfrlnTTn8pZkfpJMUgYD7YZ3qzeJgWUn8XfVYBARc4wnmNOmLbmuuaAs3q5fvB0UJOn6yHAKaGTPM7d6ezoD/A== + dependencies: + available-typed-arrays "^1.0.5" + call-bind "^1.0.2" + es-abstract "^1.20.0" + for-each "^0.3.3" + has-tostringtag "^1.0.0" + +is-typedarray@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= +is-weakref@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" + integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== + dependencies: + call-bind "^1.0.2" + is-yarn-global@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232" @@ -1893,35 +1822,17 @@ isarray@^1.0.0, isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= - -js-yaml@^3.14.0: - version "3.14.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482" - integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= +jmespath@0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" + integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== -json-bigint@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-0.3.1.tgz#0c1729d679f580d550899d6a2226c228564afe60" - integrity sha512-DGWnSzmusIreWlEupsUelHrhwmPPE+FiQvg+drKfk2p+bdEYa5mp4PJ8JsCWqae0M2jQNb0HPvnwvf1qOTThzQ== +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== dependencies: - bignumber.js "^9.0.0" + argparse "^2.0.1" json-bigint@^1.0.0: version "1.0.0" @@ -1935,21 +1846,6 @@ json-buffer@3.0.0: resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg= -json-schema-traverse@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" - integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== - -json-schema@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" - integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= - -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= - json5@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43" @@ -1966,20 +1862,10 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - -jssha@^2.3.1: - version "2.4.2" - resolved "https://registry.yarnpkg.com/jssha/-/jssha-2.4.2.tgz#d950b095634928bd6b2bda1d42da9a3a762d65e9" - integrity sha512-/jsi/9C0S70zfkT/4UlKQa5E1xKurDnXcQizcww9JSR/Fv+uIbWM2btG+bFcL3iNoK9jIGS0ls9HWLr1iw0kFg== +jssha@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/jssha/-/jssha-3.2.0.tgz#88ec50b866dd1411deaddbe6b3e3692e4c710f16" + integrity sha512-QuruyBENDWdN4tZwJbQq7/eAK85FqrI4oDbXjy5IBhYD+2pTJyBUWZe8ctWaCkrV0gy6AaelgOZZBMeswEa/6Q== jwa@^2.0.0: version "2.0.0" @@ -1998,10 +1884,10 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -kafkajs@^1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-1.15.0.tgz#a5ada0d933edca2149177393562be6fb0875ec3a" - integrity sha512-yjPyEnQCkPxAuQLIJnY5dI+xnmmgXmhuOQ1GVxClG5KTOV/rJcW1qA3UfvyEJKTp/RTSqQnUR3HJsKFvHyTpNg== +kafkajs@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.0.2.tgz#cdfc8f57aa4fd69f6d9ca1cce4ee89bbc2a3a1f9" + integrity sha512-g6CM3fAenofOjR1bfOAqeZUEaSGhNtBscNokybSdW1rmIKYNwBPC9xQzwulFJm36u/xcxXUiCl/L/qfslapihA== keyv@^3.0.0: version "3.1.0" @@ -2015,7 +1901,7 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -latest-version@^5.0.0: +latest-version@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" integrity sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA== @@ -2030,47 +1916,25 @@ levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -locate-path@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" - integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== - dependencies: - p-locate "^4.1.0" - -lodash.at@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.at/-/lodash.at-4.6.0.tgz#93cdce664f0a1994ea33dd7cd40e23afd11b0ff8" - integrity sha1-k83OZk8KGZTqM9181A4jr9EbD/g= - lodash.camelcase@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY= -lodash.has@^4.5.2: - version "4.5.2" - resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862" - integrity sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI= - lodash.snakecase@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz#39d714a35357147837aefd64b5dcbb16becd8f8d" integrity sha1-OdcUo1NXFHg3rv1ktdy7Fr7Nj40= -lodash@^4.17.19: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -logform@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/logform/-/logform-2.2.0.tgz#40f036d19161fc76b68ab50fdc7fe495544492f2" - integrity sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg== +logform@^2.3.2, logform@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.4.0.tgz#131651715a17d50f09c2a2c1a524ff1a4164bcfe" + integrity sha512-CPSJw4ftjf517EhXZGGvTHHkYobo7ZCc0kvwUoOYcjfR2UVrI66RHj8MCrfAdEitdmFqbu2BYdYs8FHHZSb6iw== dependencies: - colors "^1.2.1" - fast-safe-stringify "^2.0.4" + "@colors/colors" "1.5.0" fecha "^4.2.0" ms "^2.1.1" + safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" long@^4.0.0: @@ -2078,6 +1942,11 @@ long@^4.0.0: resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== +long@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== + lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" @@ -2112,7 +1981,7 @@ merge-descriptors@1.0.1: resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= -merge2@^1.3.0: +merge2@^1.3.0, merge2@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== @@ -2122,13 +1991,13 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= -micromatch@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259" - integrity sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q== +micromatch@^4.0.4: + version "4.0.5" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" + integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== dependencies: - braces "^3.0.1" - picomatch "^2.0.5" + braces "^3.0.2" + picomatch "^2.3.1" mime-db@1.44.0: version "1.44.0" @@ -2140,7 +2009,12 @@ mime-db@1.48.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== -mime-types@^2.1.12, mime-types@~2.1.19: +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== @@ -2154,16 +2028,18 @@ mime-types@~2.1.24: dependencies: mime-db "1.48.0" +mime-types@~2.1.34: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mime@^2.2.0: - version "2.4.6" - resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" - integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA== - mimic-response@^1.0.0, mimic-response@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" @@ -2181,35 +2057,30 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6: + version "1.2.6" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -moment@^2.11.2: - version "2.27.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" - integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== - -mpns@2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/mpns/-/mpns-2.1.3.tgz#4328bb23ca79669e3383389074c898713e22ccb9" - integrity sha512-gPLNoVqwYoKUmNYZ2shMSdaE2XvHSRxWNzyG4DUi6Av7MSujyeOw/nj61nnQeuV/vke5E0Dni468xn0qxTHIZQ== +moment@^2.29.1: + version "2.29.3" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" + integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" - integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== +ms@2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== ms@^2.1.1: version "2.1.2" @@ -2229,53 +2100,45 @@ napi-build-utils@^1.0.1: resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== -negotiator@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" - integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== +negotiator@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" + integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -node-abi@^2.7.0: +node-abi@^2.21.0: version "2.30.1" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.30.1.tgz#c437d4b1fe0e285aaf290d45b45d4d7afedac4cf" integrity sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w== dependencies: semver "^5.4.1" -node-fetch@^2.3.0, node-fetch@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@^2.6.1: - version "2.6.2" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.2.tgz#986996818b73785e47b1965cc34eb093a1d464d0" - integrity sha512-aLoxToI6RfZ+0NOjmWAgn9+LEd30YCkJKFSyWacNZdEKTit/ZMcKjGkTRo8uWEsnIb/hfKecNPEbln02PdWbcA== +node-fetch@^2.6.1, node-fetch@^2.6.6, node-fetch@^2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" -node-forge@^0.9.0: - version "0.9.1" - resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.1.tgz#775368e6846558ab6676858a4d8c6e8d16c677b5" - integrity sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ== +node-forge@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" + integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== -nodemon@^2.0.12: - version "2.0.12" - resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.12.tgz#5dae4e162b617b91f1873b3bfea215dd71e144d5" - integrity sha512-egCTmNZdObdBxUBw6ZNwvZ/xzk24CKRs5K6d+5zbmrMr7rOpPmfPeF6OxM3DDpaRx331CQRFEktn+wrFFfBSOA== +nodemon@^2.0.16: + version "2.0.16" + resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.16.tgz#d71b31bfdb226c25de34afea53486c8ef225fdef" + integrity sha512-zsrcaOfTWRuUzBn3P44RDliLlp263Z/76FPoHFr3cFFkOz0lTPAcIw8dCzfdVIx/t3AtDYCZRCDkoCojJqaG3w== dependencies: - chokidar "^3.2.2" - debug "^3.2.6" + chokidar "^3.5.2" + debug "^3.2.7" ignore-by-default "^1.0.1" minimatch "^3.0.4" - pstree.remy "^1.1.7" + pstree.remy "^1.1.8" semver "^5.7.1" supports-color "^5.5.0" touch "^3.1.0" - undefsafe "^2.0.3" - update-notifier "^4.1.0" - -noop-logger@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" - integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI= + undefsafe "^2.0.5" + update-notifier "^5.1.0" nopt@~1.0.10: version "1.0.10" @@ -2309,11 +2172,6 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= -oauth-sign@~0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" - integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== - object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -2324,10 +2182,35 @@ object-hash@^2.0.1: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea" integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= +object-hash@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-3.0.0.tgz#73f97f753e7baffc0e2cc9d6e079079744ac82e9" + integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw== + +object-inspect@^1.12.0, object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + +object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object.assign@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" + integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + has-symbols "^1.0.1" + object-keys "^1.1.1" + +on-finished@2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" @@ -2372,25 +2255,6 @@ p-is-promise@^3.0.0: resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971" integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ== -p-limit@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" - integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== - dependencies: - p-try "^2.0.0" - -p-locate@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" - integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== - dependencies: - p-limit "^2.2.0" - -p-try@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" - integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== - package-json@^6.3.0: version "6.5.0" resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" @@ -2406,15 +2270,10 @@ parseurl@~1.3.3: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -path-exists@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" - integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== - -path-parse@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" - integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-to-regexp@0.1.7: version "0.1.7" @@ -2426,54 +2285,54 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= - -picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1: +picomatch@^2.0.4, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== -pkg-fetch@3.2.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.2.2.tgz#33f391eb176c1844e93189a32f2279b36a1ec949" - integrity sha512-bLhFNT4cNnONxzbHo1H2mCCKuQkCR4dgQtv0gUZnWtp8TDP0v0UAXKHG7DXhAoTC5IYP3slLsFJtIda9ksny8g== +picomatch@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + +pkg-fetch@3.4.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.4.1.tgz#be68bb9f7fdb0f6ed995abc518ab2e35aa64d2fd" + integrity sha512-fS4cdayCa1r4jHkOKGPJKnS9PEs6OWZst+s+m0+CmhmPZObMnxoRnf9T9yUWl+lzM2b5aJF7cnQIySCT7Hq8Dg== dependencies: - chalk "^4.1.0" + chalk "^4.1.2" fs-extra "^9.1.0" https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" + node-fetch "^2.6.6" progress "^2.0.3" semver "^7.3.5" + tar-fs "^2.1.1" yargs "^16.2.0" -pkg@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.3.1.tgz#8f81671613b9e5bb1d83c39b2eed4799e1e679fe" - integrity sha512-jT/sptM1ZG++FNk+jnJYNoWLDQXYd7hqpnBhd5j18SNW1jJzNYo55RahuCiD0KN0PX9mb53GWCqKM0ia/mJytA== +pkg@^5.7.0: + version "5.7.0" + resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.7.0.tgz#6422df05e8aa147764be6ef912921d0fa719ea95" + integrity sha512-PTiAjNq/CGAtK5qUBR6pjheqnipTFjeecgSgIKEcAOJA4GpmZeOZC8pMOoT0rfes5vHsmcFo7wbSRTAmXQurrg== dependencies: - "@babel/parser" "7.13.13" - "@babel/types" "7.13.12" - chalk "^4.1.0" + "@babel/parser" "7.17.10" + "@babel/types" "7.17.10" + chalk "^4.1.2" escodegen "^2.0.0" fs-extra "^9.1.0" - globby "^11.0.3" + globby "^11.1.0" into-stream "^6.0.0" - minimist "^1.2.5" + is-core-module "2.9.0" + minimist "^1.2.6" multistream "^4.1.0" - pkg-fetch "3.2.2" - prebuild-install "6.0.1" - progress "^2.0.3" - resolve "^1.20.0" + pkg-fetch "3.4.1" + prebuild-install "6.1.4" + resolve "^1.22.0" stream-meter "^1.0.4" - tslib "2.1.0" -prebuild-install@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-6.0.1.tgz#5902172f7a40eb67305b96c2a695db32636ee26d" - integrity sha512-7GOJrLuow8yeiyv75rmvZyeMGzl8mdEX5gY69d6a6bHWmiPevwqFw+tQavhK0EYMaSg3/KD24cWqeQv1EWsqDQ== +prebuild-install@6.1.4: + version "6.1.4" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-6.1.4.tgz#ae3c0142ad611d58570b89af4986088a4937e00f" + integrity sha512-Z4vpywnK1lBg+zdPCVCsKq0xO66eEV9rWo2zrROGGiRS4JtueBOdlB1FnY8lcy7JsUud/Q3ijUxyWN26Ika0vQ== dependencies: detect-libc "^1.0.3" expand-template "^2.0.3" @@ -2481,15 +2340,13 @@ prebuild-install@6.0.1: minimist "^1.2.3" mkdirp-classic "^0.5.3" napi-build-utils "^1.0.1" - node-abi "^2.7.0" - noop-logger "^0.1.1" + node-abi "^2.21.0" npmlog "^4.0.1" pump "^3.0.0" rc "^1.2.7" simple-get "^3.0.3" tar-fs "^2.0.0" tunnel-agent "^0.6.0" - which-pm-runs "^1.0.0" prelude-ls@~1.1.2: version "1.1.2" @@ -2516,10 +2373,17 @@ progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -protobufjs@^6.8.6, protobufjs@^6.9.0: - version "6.10.1" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.10.1.tgz#e6a484dd8f04b29629e9053344e3970cccf13cd2" - integrity sha512-pb8kTchL+1Ceg4lFd5XUpK8PdWacbvV5SK2ULH2ebrYtl4GjJmS24m6CKME67jzV53tbJxHlnNOSqQHbTsR9JQ== +proto3-json-serializer@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/proto3-json-serializer/-/proto3-json-serializer-1.0.1.tgz#5d2b0b3c85568edb62984c94ce2e726985de44be" + integrity sha512-jtnGKL8EE1mTOl2qgMZJQXbS22dlb2K07i4b5vp8yMGMJ6mFSgBg4Ossu/g9NCuamdYXHjp3XxyWw4tJ0G/uKw== + dependencies: + protobufjs "^6.11.3" + +protobufjs@6.11.3, protobufjs@^6.11.3: + version "6.11.3" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.3.tgz#637a527205a35caa4f3e2a9a4a13ddffe0e7af74" + integrity sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" @@ -2532,10 +2396,10 @@ protobufjs@^6.8.6, protobufjs@^6.9.0: "@protobufjs/pool" "^1.1.0" "@protobufjs/utf8" "^1.1.0" "@types/long" "^4.0.1" - "@types/node" "^13.7.0" + "@types/node" ">=13.7.0" long "^4.0.0" -proxy-addr@~2.0.5: +proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== @@ -2543,12 +2407,12 @@ proxy-addr@~2.0.5: forwarded "0.2.0" ipaddr.js "1.9.1" -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== -pstree.remy@^1.1.7: +pstree.remy@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w== @@ -2566,27 +2430,24 @@ punycode@1.3.2: resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= -punycode@^2.1.0, punycode@^2.1.1: +punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -pupa@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.0.1.tgz#dbdc9ff48ffbea4a26a069b6f9f7abb051008726" - integrity sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA== +pupa@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" + integrity sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A== dependencies: escape-goat "^2.0.0" -qs@6.7.0: - version "6.7.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" - integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== - -qs@~6.5.2: - version "6.5.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" - integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +qs@6.10.3: + version "6.10.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.3.tgz#d6cde1b2ffca87b5aa57889816c5f81535e22e8e" + integrity sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ== + dependencies: + side-channel "^1.0.4" querystring@0.2.0: version "0.2.0" @@ -2603,13 +2464,13 @@ range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332" - integrity sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q== +raw-body@2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: - bytes "3.1.0" - http-errors "1.7.2" + bytes "3.1.2" + http-errors "2.0.0" iconv-lite "0.4.24" unpipe "1.0.0" @@ -2633,7 +2494,7 @@ rc@^1.2.7, rc@^1.2.8: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.3.7: +readable-stream@^2.0.0, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.3.7: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -2655,13 +2516,22 @@ readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" -readdirp@~3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.4.0.tgz#9fdccdf9e9155805449221ac645e8303ab5b9ada" - integrity sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ== +readdirp@~3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" + integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" +regexp.prototype.flags@^1.4.3: + version "1.4.3" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac" + integrity sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + functions-have-names "^1.2.2" + registry-auth-token@^4.0.0: version "4.2.0" resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.0.tgz#1d37dffda72bbecd0f581e4715540213a65eb7da" @@ -2676,54 +2546,24 @@ registry-url@^5.0.0: dependencies: rc "^1.2.8" -request@^2.81.0: - version "2.88.2" - resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" - integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - har-validator "~5.1.3" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - oauth-sign "~0.9.0" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.5.0" - tunnel-agent "^0.6.0" - uuid "^3.3.2" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= -require-main-filename@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" - integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== - requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= -resolve@^1.20.0: - version "1.20.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" - integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== +resolve@^1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198" + integrity sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw== dependencies: - is-core-module "^2.2.0" - path-parse "^1.0.6" + is-core-module "^2.8.1" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" responselike@^1.0.2: version "1.0.2" @@ -2732,31 +2572,32 @@ responselike@^1.0.2: dependencies: lowercase-keys "^1.0.0" -retry-request@^4.0.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.1.3.tgz#d5f74daf261372cff58d08b0a1979b4d7cab0fde" - integrity sha512-QnRZUpuPNgX0+D1xVxul6DbJ9slvo4Rm6iV/dn63e048MvGbUZiKySVt6Tenp04JqmchxjiLltGerOJys7kJYQ== +retry-request@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-5.0.0.tgz#886ff8ec0e77fffbe66a4d5e90fd8f6646b6eae4" + integrity sha512-vBZdBxUordje9253imlmGtppC5gdcwZmNz7JnU2ui+KKFPk25keR+0c020AVV20oesYxIFOI0Kh3HE88/59ieg== dependencies: debug "^4.1.1" + extend "^3.0.2" reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rhea-promise@^0.1.15: - version "0.1.15" - resolved "https://registry.yarnpkg.com/rhea-promise/-/rhea-promise-0.1.15.tgz#00175324352224424d59b7712faf14097a84e8f2" - integrity sha512-+6uilZXSJGyiqVeHQI3Krv6NTAd8cWRCY2uyCxmzR4/5IFtBqqFem1HV2OiwSj0Gu7OFChIJDfH2JyjN7J0vRA== +rhea-promise@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/rhea-promise/-/rhea-promise-2.1.0.tgz#540e97f62302ad54b368b79a3560a51eefd2b063" + integrity sha512-CRMwdJ/o4oO/xKcvAwAsd0AHy5fVvSlqso7AadRmaaLGzAzc9LCoW7FOFnucI8THasVmOeCnv5c/fH/n7FcNaA== dependencies: debug "^3.1.0" - rhea "^1.0.4" - tslib "^1.9.3" + rhea "^2.0.3" + tslib "^2.2.0" -rhea@^1.0.18, rhea@^1.0.23, rhea@^1.0.4: - version "1.0.24" - resolved "https://registry.yarnpkg.com/rhea/-/rhea-1.0.24.tgz#7084e4c635aa8ba0faf2d9602ac9f97d26aedc00" - integrity sha512-PEl62U2EhxCO5wMUZ2/bCBcXAVKN9AdMSNQOrp3+R5b77TEaOSiy16MQ0sIOmzj/iqsgIAgPs1mt3FYfu1vIXA== +rhea@^2.0.3: + version "2.0.8" + resolved "https://registry.yarnpkg.com/rhea/-/rhea-2.0.8.tgz#2ac355fad923e36a995defffdf314bb1cbe562de" + integrity sha512-IgwlP4D2lzinBSll5f35tAWa30dGCZhG9Ujd1DiaB7MUGegIjAaLzqATCw3ha+h9oq9mXcitqayBbNIXYdvtFg== dependencies: debug "0.8.0 - 3.5.0" @@ -2765,17 +2606,22 @@ run-parallel@^1.1.9: resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q== -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safe-stable-stringify@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73" + integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg== -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -2807,10 +2653,12 @@ semver@^6.0.0, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.1.3: - version "7.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" - integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== +semver@^7.3.4: + version "7.3.7" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" + integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== + dependencies: + lru-cache "^6.0.0" semver@^7.3.5: version "7.3.5" @@ -2819,44 +2667,53 @@ semver@^7.3.5: dependencies: lru-cache "^6.0.0" -send@0.17.1: - version "0.17.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" - integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== +send@0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== dependencies: debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" + depd "2.0.0" + destroy "1.2.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" - http-errors "~1.7.2" + http-errors "2.0.0" mime "1.6.0" - ms "2.1.1" - on-finished "~2.3.0" + ms "2.1.3" + on-finished "2.4.1" range-parser "~1.2.1" - statuses "~1.5.0" + statuses "2.0.1" -serve-static@1.14.1: - version "1.14.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" - integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== +serve-static@1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== dependencies: encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.17.1" + send "0.18.0" -set-blocking@^2.0.0, set-blocking@~2.0.0: +set-blocking@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= -setprototypeof@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" - integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.3" @@ -2894,43 +2751,15 @@ source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= - -sshpk@^1.7.0: - version "1.16.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" - integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - bcrypt-pbkdf "^1.0.0" - dashdash "^1.12.0" - ecc-jsbn "~0.1.1" - getpass "^0.1.1" - jsbn "~0.1.0" - safer-buffer "^2.0.2" - tweetnacl "~0.14.0" - stack-trace@0.0.x: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= -"statuses@>= 1.5.0 < 2", statuses@~1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" - integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= - -stream-browserify@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" - integrity sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg== - dependencies: - inherits "~2.0.1" - readable-stream "^2.0.2" +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== stream-meter@^1.0.4: version "1.0.4" @@ -2979,6 +2808,33 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" +string-width@^4.2.2: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string.prototype.trimend@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz#914a65baaab25fbdd4ee291ca7dde57e869cb8d0" + integrity sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.4" + es-abstract "^1.19.5" + +string.prototype.trimstart@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz#5466d93ba58cfa2134839f81d7f42437e8c01fef" + integrity sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.4" + es-abstract "^1.19.5" + string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" @@ -3026,6 +2882,13 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" @@ -3045,7 +2908,12 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -tar-fs@^2.0.0: +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -3066,21 +2934,11 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" -term-size@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753" - integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw== - text-hex@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== -through@~2.3.4: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= - to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" @@ -3098,10 +2956,10 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -toidentifier@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" - integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +toidentifier@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" + integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== touch@^3.1.0: version "3.1.0" @@ -3119,33 +2977,25 @@ tough-cookie@^4.0.0: punycode "^2.1.1" universalify "^0.1.2" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== triple-beam@^1.2.0, triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== -tslib@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" - integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== - -tslib@^1.10.0, tslib@^1.9.3: +tslib@^1.9.3: version "1.13.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" integrity sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q== -tslib@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.1.tgz#410eb0d113e5b6356490eec749603725b021b43e" - integrity sha512-SgIkNheinmEBgx1IUNirK0TUD4X9yjjBRTqqjggWCU3pUEqIk3/Uwl3yRixYKT6WjQuGiwDv4NomL3wqRCj+CQ== +tslib@^2.2.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" + integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== tunnel-agent@^0.6.0: version "0.6.0" @@ -3154,16 +3004,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -tunnel@^0.0.6, tunnel@~0.0.2: +tunnel@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= - type-check@~0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" @@ -3171,12 +3016,12 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-fest@^0.8.1: - version "0.8.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" - integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -type-is@~1.6.17, type-is@~1.6.18: +type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== @@ -3191,22 +3036,20 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -undefsafe@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.3.tgz#6b166e7094ad46313b2202da7ecc2cd7cc6e7aae" - integrity sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A== +unbox-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" + integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== dependencies: - debug "^2.2.0" - -underscore@1.4.x: - version "1.4.4" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" - integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ= + call-bind "^1.0.2" + has-bigints "^1.0.2" + has-symbols "^1.0.3" + which-boxed-primitive "^1.0.2" -underscore@^1.8.3: - version "1.10.2" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf" - integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg== +undefsafe@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" + integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== unique-string@^2.0.0: version "2.0.0" @@ -3235,32 +3078,26 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= -update-notifier@^4.1.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.3.tgz#be86ee13e8ce48fb50043ff72057b5bd598e1ea3" - integrity sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A== +update-notifier@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" + integrity sha512-ItnICHbeMh9GqUy31hFPrD1kcuZ3rpxDZbf4KUDavXwS0bW5m7SLbDQpGX3UYr072cbrF5hFUs3r5tUsPwjfHw== dependencies: - boxen "^4.2.0" - chalk "^3.0.0" + boxen "^5.0.0" + chalk "^4.1.0" configstore "^5.0.1" has-yarn "^2.1.0" import-lazy "^2.1.0" is-ci "^2.0.0" - is-installed-globally "^0.3.1" - is-npm "^4.0.0" + is-installed-globally "^0.4.0" + is-npm "^5.0.0" is-yarn-global "^0.3.0" - latest-version "^5.0.0" - pupa "^2.0.1" + latest-version "^5.1.0" + pupa "^2.1.1" + semver "^7.3.4" semver-diff "^3.1.1" xdg-basedir "^4.0.0" -uri-js@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" - integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== - dependencies: - punycode "^2.1.0" - url-parse-lax@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c" @@ -3268,10 +3105,10 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" -url-parse@~1.4.3: - version "1.4.7" - resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.7.tgz#a8a83535e8c00a316e403a5db4ac1b9b853ae278" - integrity sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg== +url-parse@~1.5.10: + version "1.5.10" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== dependencies: querystringify "^2.1.1" requires-port "^1.0.0" @@ -3297,12 +3134,17 @@ util-deprecate@^1.0.1, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -util@^0.11.1: - version "0.11.1" - resolved "https://registry.yarnpkg.com/util/-/util-0.11.1.tgz#3236733720ec64bb27f6e26f421aaa2e1b588d61" - integrity sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ== +util@^0.12.1: + version "0.12.4" + resolved "https://registry.yarnpkg.com/util/-/util-0.12.4.tgz#66121a31420df8f01ca0c464be15dfa1d1850253" + integrity sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw== dependencies: - inherits "2.0.3" + inherits "^2.0.3" + is-arguments "^1.0.4" + is-generator-function "^1.0.7" + is-typed-array "^1.1.3" + safe-buffer "^5.1.2" + which-typed-array "^1.1.2" utils-merge@1.0.1: version "1.0.1" @@ -3319,54 +3161,56 @@ uuid-random@^1.3.2: resolved "https://registry.yarnpkg.com/uuid-random/-/uuid-random-1.3.2.tgz#96715edbaef4e84b1dcf5024b00d16f30220e2d0" integrity sha512-UOzej0Le/UgkbWEO8flm+0y+G+ljUon1QWTEZOq1rnMAsxo2+SckbiZdKzAHHlVh6gJqI1TjC/xwgR50MuCrBQ== -uuid@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== - -uuid@^3.3.2: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== - -uuid@^8.1.0: - version "8.3.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.0.tgz#ab738085ca22dc9a8c92725e459b1d507df5d6ea" - integrity sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ== +uuid@8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" + integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== -validator@^9.4.1: - version "9.4.1" - resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.1.tgz#abf466d398b561cd243050112c6ff1de6cc12663" - integrity sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA== +uuid@^8.3.0: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== -walkdir@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/walkdir/-/walkdir-0.4.1.tgz#dc119f83f4421df52e3061e514228a2db20afa39" - integrity sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" -which-module@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" - integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= +which-boxed-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" + integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== + dependencies: + is-bigint "^1.0.1" + is-boolean-object "^1.1.0" + is-number-object "^1.0.4" + is-string "^1.0.5" + is-symbol "^1.0.3" -which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" - integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= +which-typed-array@^1.1.2: + version "1.1.8" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.8.tgz#0cfd53401a6f334d90ed1125754a42ed663eb01f" + integrity sha512-Jn4e5PItbcAHyLoRDwvPj1ypu27DJbtdYXUa5zsinrUx77Uvfb0cXwwnGMTn7cjUfhhqgVQnVJCwF+7cgU7tpw== + dependencies: + available-typed-arrays "^1.0.5" + call-bind "^1.0.2" + es-abstract "^1.20.0" + for-each "^0.3.3" + has-tostringtag "^1.0.0" + is-typed-array "^1.1.9" wide-align@^1.1.0: version "1.1.3" @@ -3382,17 +3226,17 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" -winston-daily-rotate-file@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.5.0.tgz#3914ac57c4bdae1138170bec85af0c2217b253b1" - integrity sha512-/HqeWiU48dzGqcrABRlxYWVMdL6l3uKCtFSJyrqK+E2rLnSFNsgYpvwx15EgTitBLNzH69lQd/+z2ASryV2aqw== +winston-daily-rotate-file@^4.7.1: + version "4.7.1" + resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz#f60a643af87f8867f23170d8cd87dbe3603a625f" + integrity sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA== dependencies: - file-stream-rotator "^0.5.7" + file-stream-rotator "^0.6.1" object-hash "^2.0.1" triple-beam "^1.3.0" - winston-transport "^4.2.0" + winston-transport "^4.4.0" -winston-transport@^4.2.0, winston-transport@^4.4.0: +winston-transport@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.4.0.tgz#17af518daa690d5b2ecccaa7acf7b20ca7925e59" integrity sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw== @@ -3400,40 +3244,36 @@ winston-transport@^4.2.0, winston-transport@^4.4.0: readable-stream "^2.3.7" triple-beam "^1.2.0" -winston@^3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/winston/-/winston-3.3.3.tgz#ae6172042cafb29786afa3d09c8ff833ab7c9170" - integrity sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw== +winston-transport@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.5.0.tgz#6e7b0dd04d393171ed5e4e4905db265f7ab384fa" + integrity sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q== + dependencies: + logform "^2.3.2" + readable-stream "^3.6.0" + triple-beam "^1.3.0" + +winston@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.7.2.tgz#95b4eeddbec902b3db1424932ac634f887c400b1" + integrity sha512-QziIqtojHBoyzUOdQvQiar1DH0Xp9nF1A1y7NVy2DGEsz82SBDtOalS0ulTRGVT14xPX3WRWkCsdcJKqNflKng== dependencies: "@dabh/diagnostics" "^2.0.2" - async "^3.1.0" + async "^3.2.3" is-stream "^2.0.0" - logform "^2.2.0" + logform "^2.4.0" one-time "^1.0.0" readable-stream "^3.4.0" + safe-stable-stringify "^2.3.1" stack-trace "0.0.x" triple-beam "^1.3.0" - winston-transport "^4.4.0" - -wns@~0.5.3: - version "0.5.4" - resolved "https://registry.yarnpkg.com/wns/-/wns-0.5.4.tgz#ad8e2ee60e675557da9610d94444a7f59eceaf78" - integrity sha512-WYiJ7khIwUGBD5KAm+YYmwJDDRzFRs4YGAjtbFSoRIdbn9Jcix3p9khJmpvBTXGommaKkvduAn+pc9l4d9yzVQ== + winston-transport "^4.5.0" word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wrap-ansi@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -3479,11 +3319,6 @@ xml2js@^0.4.19: sax ">=0.6.0" xmlbuilder "~11.0.0" -xmlbuilder@15.1.1: - version "15.1.1" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" - integrity sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg== - xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" @@ -3494,11 +3329,6 @@ xmlbuilder@~9.0.1: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= -y18n@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" - integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== - y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" @@ -3509,36 +3339,11 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== -yargs-parser@^18.1.2: - version "18.1.3" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0" - integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ== - dependencies: - camelcase "^5.0.0" - decamelize "^1.2.0" - yargs-parser@^20.2.2: version "20.2.9" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== -yargs@^15.3.1: - version "15.4.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8" - integrity sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A== - dependencies: - cliui "^6.0.0" - decamelize "^1.2.0" - find-up "^4.1.0" - get-caller-file "^2.0.1" - require-directory "^2.1.1" - require-main-filename "^2.0.0" - set-blocking "^2.0.0" - string-width "^4.2.0" - which-module "^2.0.0" - y18n "^4.0.0" - yargs-parser "^18.1.2" - yargs@^16.2.0: version "16.2.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" From 86b88c2699cfb38b9460fa5a57b41198b1035c31 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 14 Jun 2022 11:18:38 +0300 Subject: [PATCH 211/262] Fix compilation errors. UI: Add entity view VC support. --- .../ie/importing/impl/EntityViewImportService.java | 11 +++++++---- .../pages/entity-view/entity-view-tabs.component.html | 6 ++++++ ui-ngx/src/app/shared/models/vc.models.ts | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java index 9bbb57fb4f..20ffbb08d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -20,14 +20,15 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @TbCoreComponent @@ -43,15 +44,17 @@ public class EntityViewImportService extends BaseEntityImportService exportData, IdProvider idProvider, EntityImportSettings importSettings) { + protected EntityView prepareAndSave(EntitiesImportCtx ctx, EntityView entityView, EntityExportData exportData, IdProvider idProvider) { entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); return entityViewService.saveEntityView(entityView); } @Override protected void onEntitySaved(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { - entityNotificationService.notifyCreateOrUpdateEntity(user.getTenantId(), savedEntityView.getId(), savedEntityView, - null, oldEntityView == null ? ActionType.ADDED : ActionType.UPDATED, user); + super.onEntitySaved(user, savedEntityView, oldEntityView); + if (oldEntityView != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedEntityView.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html index 7ba3c80805..79307ddcb2 100644 --- a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index b0485cc37e..d5282c43f7 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -24,6 +24,7 @@ import { RuleChain, RuleChainMetaData } from '@shared/models/rule-chain.models'; export const exportableEntityTypes: Array = [ EntityType.ASSET, EntityType.DEVICE, + EntityType.ENTITY_VIEW, EntityType.DASHBOARD, EntityType.CUSTOMER, EntityType.DEVICE_PROFILE, From 389ed5594e7f7b6b923e7fac3a62ccad058a0900 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 14 Jun 2022 11:31:30 +0300 Subject: [PATCH 212/262] UI: Fix duplicate enum --- ui-ngx/src/app/shared/models/vc.models.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index d5282c43f7..5ef0a18b08 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -29,7 +29,6 @@ export const exportableEntityTypes: Array = [ EntityType.CUSTOMER, EntityType.DEVICE_PROFILE, EntityType.RULE_CHAIN, - EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE ]; From eab4e333a9c6d4fe487343c8004484d12c0652ff Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Jun 2022 12:03:05 +0300 Subject: [PATCH 213/262] refactoring: alarm - add Mockito to create --- .../controller/BaseAlarmControllerTest.java | 83 ++++++++++++++++++- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java index d82f4175b5..ee1a952546 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java @@ -128,8 +128,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testUpdateAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + alarm.setSeverity(AlarmSeverity.MAJOR); loginDifferentTenant(); @@ -144,8 +150,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testUpdateAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + loginDifferentCustomer(); alarm.setSeverity(AlarmSeverity.MAJOR); @@ -160,10 +172,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testDeleteAlarmViaCustomer() throws Exception { loginCustomerUser(); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); testNotifyEntityOneMsgToEdgeServiceNever(alarm, alarm.getId(), alarm.getOriginator(), @@ -174,10 +190,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testDeleteAlarmViaTenant() throws Exception { loginTenantAdmin(); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); testNotifyEntityOneMsgToEdgeServiceNever(alarm, alarm.getId(), alarm.getOriginator(), @@ -188,8 +208,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testDeleteAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); @@ -204,7 +230,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testDeleteAlarmViaAnotherCustomer() throws Exception { loginCustomerUser(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -219,10 +252,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testClearAlarmViaCustomer() throws Exception { loginCustomerUser(); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); Alarm foundAlarm = doGet("/api/alarm/" + alarm.getId(), Alarm.class); @@ -237,8 +274,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testClearAlarmViaTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); @@ -254,10 +297,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testAcknowledgeAlarmViaCustomer() throws Exception { loginCustomerUser(); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + doPost("/api/alarm/" + alarm.getId() + "/ack").andExpect(status().isOk()); Alarm foundAlarm = doGet("/api/alarm/" + alarm.getId(), Alarm.class); @@ -272,7 +319,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testClearAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -286,7 +340,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testClearAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); @@ -300,7 +361,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testAcknowledgeAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -314,7 +382,14 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { @Test public void testAcknowledgeAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Alarm alarm = createAlarm(TEST_ALARM_TYPE); + + testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); From 8bccd857ebd741ade07e3c5f82ea2e5de83ea56c Mon Sep 17 00:00:00 2001 From: fe-dev Date: Tue, 14 Jun 2022 12:06:46 +0300 Subject: [PATCH 214/262] UI: Add input placeholder --- .../authentication-dialog/sms-auth-dialog.component.html | 6 +++++- ui-ngx/src/app/shared/components/phone-input.component.html | 4 ++-- ui-ngx/src/app/shared/components/phone-input.component.ts | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html index bb547f56a3..e216c22d3e 100644 --- a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html @@ -37,7 +37,11 @@

security.2fa.dialog.sms-step-description

- + +
- security.2fa.dialog.sms-step-label + {{ placeholder | translate }} = this.countryCodeData.allCountries; phonePlaceholder: string; From 1838e147a6f5c815edd832e81a6e25da8f869e96 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Jun 2022 12:28:37 +0300 Subject: [PATCH 215/262] Batch Relations save for import performance improvement --- .../DefaultEntitiesExportImportService.java | 26 +++- .../sync/ie/EntitiesExportImportService.java | 4 +- .../impl/BaseEntityImportService.java | 29 +++-- .../DefaultEntitiesVersionControlService.java | 26 ++-- .../sync/vc/data/EntitiesImportCtx.java | 11 ++ .../server/dao/relation/RelationService.java | 3 + .../dao/relation/BaseRelationService.java | 17 +++ .../server/dao/relation/RelationDao.java | 3 + .../AbstractRelationInsertRepository.java | 51 -------- .../HsqlRelationInsertRepository.java | 63 ---------- .../dao/sql/relation/JpaRelationDao.java | 8 ++ .../PsqlRelationInsertRepository.java | 41 ------ .../relation/RelationInsertRepository.java | 4 + .../relation/SqlRelationInsertRepository.java | 118 ++++++++++++++++++ 14 files changed, 222 insertions(+), 182 deletions(-) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 5ef7b84ce7..334a1349a6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -21,15 +21,19 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.apiusage.RateLimitService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; @@ -38,6 +42,7 @@ import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExport import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; @@ -53,6 +58,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); + private final EntityActionService entityActionService; + private final RelationService relationService; private final RateLimitService rateLimitService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( @@ -76,7 +83,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, - boolean saveReferences, boolean sendEvents) throws ThingsboardException { + boolean saveReferencesAndSendEvents) throws ThingsboardException { if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } @@ -89,17 +96,28 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityImportResult importResult = importService.importEntity(ctx, exportData); - if (saveReferences) { + if (saveReferencesAndSendEvents) { importResult.getSaveReferencesCallback().run(); - } - if (sendEvents) { importResult.getSendEventsCallback().run(); + saveRelations(ctx); } ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); return importResult; } + @Override + public void saveRelations(EntitiesImportCtx ctx) throws ThingsboardException { + relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); + + for (EntityRelation relation : ctx.getRelations()) { + entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + } + } + @Override public Comparator getEntityTypeComparatorForImport() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 221b29622c..f0f41c7cdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -33,9 +33,11 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, - boolean saveReferences, boolean sendEvents) throws ThingsboardException; + boolean saveReferencesAndSendEvents) throws ThingsboardException; + void saveRelations(EntitiesImportCtx ctx) throws ThingsboardException; + Comparator getEntityTypeComparatorForImport(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index f1cef3191c..2c1fd77248 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.google.api.client.util.Objects; import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -55,9 +56,12 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -110,6 +114,11 @@ public abstract class BaseEntityImportService relationsMap = new LinkedHashMap<>(); + relations.forEach(r -> relationsMap.put(r, r)); + if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { - if (!relations.contains(existingRelation)) { + EntityRelation relation = relationsMap.get(existingRelation); + if (relation == null) { relationService.deleteRelation(tenantId, existingRelation); importResult.addSendEventsCallback(() -> { entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, @@ -158,19 +171,15 @@ public abstract class BaseEntityImportService { - entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); - entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); - }); - } + ctx.addRelations(relationsMap.values()); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c7bae4b8df..fb986cdd2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -69,6 +70,7 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; @@ -112,6 +114,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final TbNotificationEntityService entityNotificationService; + private final RelationService relationService; private final TransactionTemplate transactionTemplate; private ListeningExecutorService executor; @@ -252,7 +255,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .saveCredentials(config.isLoadCredentials()) .findExistingByName(false) .build()); - EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, true, true); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, true); + return VersionLoadResult.success(EntityTypeLoadResult.builder() .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) @@ -300,7 +304,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { - importResult = exportImportService.importEntity(ctx, entityData, false, false); + importResult = exportImportService.importEntity(ctx, entityData, false); } catch (Exception e) { throw new LoadEntityException(entityData, e); } @@ -332,7 +336,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get(); importSettings.setResetExternalIdsOfAnotherTenant(true); ctx.setSettings(importSettings); - EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, false, false); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, false); EntityTypeLoadResult stats = results.get(externalId.getEntityType()); if (importResult.getOldEntity() == null) stats.setCreated(stats.getCreated() + 1); @@ -369,19 +373,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont sw.startNew("Callbacks"); - for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { - try { + try { + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { saveReferencesCallback.run(); - } catch (ThingsboardException e) { - throw new RuntimeException(e); } - } - for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { - try { + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); } + sw.startNew("Relations"); + exportImportService.saveRelations(ctx); + } catch (ThingsboardException e) { + throw new RuntimeException(e); } sw.stop(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 93de254d2a..ec7c9c4fdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -17,13 +17,18 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; @Slf4j @Data @@ -33,6 +38,7 @@ public class EntitiesImportCtx { private EntityImportSettings settings; private final Map externalToInternalIdMap = new HashMap<>(); + private final Set relations = new LinkedHashSet<>(); public EntitiesImportCtx(SecurityUser user) { this(user, null); @@ -77,4 +83,9 @@ public class EntitiesImportCtx { log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId); externalToInternalIdMap.put(externalId, internalId); } + + public void addRelations(Collection values) { + relations.addAll(values); + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index dbd641c1c8..bc9980fcd8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -40,6 +41,8 @@ public interface RelationService { boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, List relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 912a3393c4..87cc858f3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -40,12 +41,14 @@ import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.sql.JpaExecutorService; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -119,6 +122,20 @@ public class BaseRelationService implements RelationService { return result; } + @Override + public void saveRelations(TenantId tenantId, List relations) { + log.trace("Executing saveRelations [{}]", relations); + for (EntityRelation relation : relations) { + validate(relation); + } + for (List partition : Lists.partition(relations, 1024)) { + relationDao.saveRelations(tenantId, partition); + } + for (EntityRelation relation : relations) { + publishEvictEvent(EntityRelationEvent.from(relation)); + } + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelationAsync [{}]", relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index e15f4f3d69..9ed3a5672f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -43,6 +44,8 @@ public interface RelationDao { boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, Collection relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java deleted file mode 100644 index ec6df53ae2..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.jpa.repository.Modifying; -import org.thingsboard.server.dao.model.sql.RelationEntity; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; - -@Slf4j -public abstract class AbstractRelationInsertRepository implements RelationInsertRepository { - - @PersistenceContext - protected EntityManager entityManager; - - protected Query getQuery(RelationEntity entity, String query) { - Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); - if (entity.getAdditionalInfo() == null) { - nativeQuery.setParameter("additionalInfo", null); - } else { - nativeQuery.setParameter("additionalInfo", entity.getAdditionalInfo().toString()); - } - return nativeQuery - .setParameter("fromId", entity.getFromId()) - .setParameter("fromType", entity.getFromType()) - .setParameter("toId", entity.getToId()) - .setParameter("toType", entity.getToType()) - .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) - .setParameter("relationType", entity.getRelationType()); - } - - @Modifying - protected abstract RelationEntity processSaveOrUpdate(RelationEntity entity); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java deleted file mode 100644 index 5da0a8a2de..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sql.RelationCompositeKey; -import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.HsqlDao; - -import javax.persistence.Query; - -@HsqlDao -@Repository -@Transactional -public class HsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { - - private static final String INSERT_ON_CONFLICT_DO_UPDATE = "MERGE INTO relation USING (VALUES :fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) R " + - "(from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " + - "ON (relation.from_id = UUID(R.from_id) AND relation.from_type = R.from_type AND relation.relation_type_group = R.relation_type_group AND relation.relation_type = R.relation_type AND relation.to_id = UUID(R.to_id) AND relation.to_type = R.to_type) " + - "WHEN MATCHED THEN UPDATE SET relation.additional_info = R.additional_info " + - "WHEN NOT MATCHED THEN INSERT (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) VALUES (UUID(R.from_id), R.from_type, UUID(R.to_id), R.to_type, R.relation_type_group, R.relation_type, R.additional_info)"; - - protected Query getQuery(RelationEntity entity, String query) { - Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); - if (entity.getAdditionalInfo() == null) { - nativeQuery.setParameter("additionalInfo", null); - } else { - nativeQuery.setParameter("additionalInfo", entity.getAdditionalInfo().toString()); - } - return nativeQuery - .setParameter("fromId", entity.getFromId().toString()) - .setParameter("fromType", entity.getFromType()) - .setParameter("toId", entity.getToId().toString()) - .setParameter("toType", entity.getToType()) - .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) - .setParameter("relationType", entity.getRelationType()); - } - - @Override - public RelationEntity saveOrUpdate(RelationEntity entity) { - return processSaveOrUpdate(entity); - } - - @Override - protected RelationEntity processSaveOrUpdate(RelationEntity entity) { - getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE).executeUpdate(); - return entityManager.find(RelationEntity.class, new RelationCompositeKey(entity.toData())); - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 71e36fcae1..550197c924 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -33,7 +33,9 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; /** * Created by Valerii Sosliuk on 5/29/2017. @@ -112,6 +114,12 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null; } + @Override + public void saveRelations(TenantId tenantId, Collection relations) { + List entities = relations.stream().map(RelationEntity::new).collect(Collectors.toList()); + relationInsertRepository.saveOrUpdate(entities); + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { return service.submit(() -> relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java deleted file mode 100644 index a622d19e6d..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.PsqlDao; - -@PsqlDao -@Repository -@Transactional -public class PsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { - - private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + - " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + - "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo returning *"; - - @Override - public RelationEntity saveOrUpdate(RelationEntity entity) { - return processSaveOrUpdate(entity); - } - - @Override - protected RelationEntity processSaveOrUpdate(RelationEntity entity) { - return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE).getSingleResult(); - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java index a07b11bcda..527a7f2bcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java @@ -17,8 +17,12 @@ package org.thingsboard.server.dao.sql.relation; import org.thingsboard.server.dao.model.sql.RelationEntity; +import java.util.List; + public interface RelationInsertRepository { RelationEntity saveOrUpdate(RelationEntity entity); + void saveOrUpdate(List entities); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java new file mode 100644 index 0000000000..c0b7bb7a70 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.relation; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.dao.model.sql.RelationEntity; +import org.thingsboard.server.dao.util.PsqlDao; + +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.Query; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +@PsqlDao +@Repository +@Transactional +public class SqlRelationInsertRepository implements RelationInsertRepository { + + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JPA = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo returning *"; + + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JDBC = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = ?"; + + + @PersistenceContext + protected EntityManager entityManager; + + @Autowired + protected JdbcTemplate jdbcTemplate; + + @Autowired + private TransactionTemplate transactionTemplate; + + protected Query getQuery(RelationEntity entity, String query) { + Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); + if (entity.getAdditionalInfo() == null) { + nativeQuery.setParameter("additionalInfo", null); + } else { + nativeQuery.setParameter("additionalInfo", JacksonUtil.toString(entity.getAdditionalInfo())); + } + return nativeQuery + .setParameter("fromId", entity.getFromId()) + .setParameter("fromType", entity.getFromType()) + .setParameter("toId", entity.getToId()) + .setParameter("toType", entity.getToType()) + .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) + .setParameter("relationType", entity.getRelationType()); + } + + @Override + public RelationEntity saveOrUpdate(RelationEntity entity) { + return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE_JPA).getSingleResult(); + } + + @Override + public void saveOrUpdate(List entities) { + transactionTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus status) { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RelationEntity relation = entities.get(i); + ps.setObject(1, relation.getFromId()); + ps.setString(2, relation.getFromType()); + ps.setObject(3, relation.getToId()); + ps.setString(4, relation.getToType()); + + ps.setString(5, relation.getRelationTypeGroup()); + ps.setString(6, relation.getRelationType()); + + if (relation.getAdditionalInfo() == null) { + ps.setString(7, null); + ps.setString(8, null); + } else { + String json = JacksonUtil.toString(relation.getAdditionalInfo()); + ps.setString(7, json); + ps.setString(8, json); + } + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }); + } + }); + } + +} From 58a438c73c9424567b8e7cf997f875d672cb6720 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 14 Jun 2022 12:42:32 +0300 Subject: [PATCH 216/262] Version control - handle entity view attributes update --- .../DefaultTbEntityViewService.java | 38 +++++++++++-------- .../entityView/TbEntityViewService.java | 2 + .../impl/EntityViewImportService.java | 8 ++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java index 878c6d32bb..2eee167e75 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java @@ -66,19 +66,33 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen @Override public EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException { ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + try { + EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); + this.updateEntityViewAttributes(user, savedEntityView, existingEntityView); + notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, + null, actionType, user); + return savedEntityView; + } catch (Exception e) { + notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e); + throw handleException(e); + } + } + + @Override + public void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { try { List> futures = new ArrayList<>(); - if (existingEntityView != null) { - if (existingEntityView.getKeys() != null && existingEntityView.getKeys().getAttributes() != null) { - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.CLIENT_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SERVER_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SHARED_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); + + if (oldEntityView != null) { + if (oldEntityView.getKeys() != null && oldEntityView.getKeys().getAttributes() != null) { + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user)); } - List tsKeys = existingEntityView.getKeys() != null && existingEntityView.getKeys().getTimeseries() != null ? - existingEntityView.getKeys().getTimeseries() : Collections.emptyList(); - futures.add(deleteLatestFromEntityView(existingEntityView, tsKeys, user)); + List tsKeys = oldEntityView.getKeys() != null && oldEntityView.getKeys().getTimeseries() != null ? + oldEntityView.getKeys().getTimeseries() : Collections.emptyList(); + futures.add(deleteLatestFromEntityView(oldEntityView, tsKeys, user)); } - EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); if (savedEntityView.getKeys() != null) { if (savedEntityView.getKeys().getAttributes() != null) { futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user)); @@ -94,13 +108,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen throw new RuntimeException("Failed to copy attributes to entity view", e); } } - - notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, - null, actionType, user); - - return savedEntityView; } catch (Exception e) { - notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e); throw handleException(e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java index dd5db9391b..e7e150531b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java @@ -28,6 +28,8 @@ public interface TbEntityViewService { EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException; + void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException; + void delete (EntityView entity, SecurityUser user) throws ThingsboardException; EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java index 20ffbb08d3..4bd903c110 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -27,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @@ -37,6 +40,10 @@ public class EntityViewImportService extends BaseEntityImportService Date: Tue, 14 Jun 2022 14:45:42 +0300 Subject: [PATCH 217/262] refactoring: commit1 --- .../controller/AbstractNotifyEntityTest.java | 4 - .../controller/BaseAlarmControllerTest.java | 75 +------------ .../BaseCustomerControllerTest.java | 105 +++++++++--------- 3 files changed, 57 insertions(+), 127 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 6fb7b93bae..40a3bac5de 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -56,7 +56,6 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.reset(tbClusterService, auditLogService); } - protected void testNotifyEntityDeleteOneMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { @@ -64,7 +63,6 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testLogEntityActionOne(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); testPushMsgToRuleEngineOne(entityId, tenantId); testBroadcastEntityStateChangeEventOne(entityId, tenantId); - } protected void testNotifyEntityOneMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, @@ -172,6 +170,4 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } return result; } - - } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java index ee1a952546..03d1223cc0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java @@ -87,12 +87,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); alarm.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm, Alarm.class); @@ -108,12 +105,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); alarm.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm, Alarm.class); @@ -129,13 +123,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); - alarm.setSeverity(AlarmSeverity.MAJOR); loginDifferentTenant(); @@ -151,13 +140,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testUpdateAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); - loginDifferentCustomer(); alarm.setSeverity(AlarmSeverity.MAJOR); @@ -173,12 +157,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); @@ -191,12 +172,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); doDelete("/api/alarm/" + alarm.getId()).andExpect(status().isOk()); @@ -209,13 +187,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); - loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); @@ -231,13 +204,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testDeleteAlarmViaAnotherCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); - loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -253,12 +221,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); @@ -275,13 +240,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); - Mockito.reset(tbClusterService, auditLogService); doPost("/api/alarm/" + alarm.getId() + "/clear").andExpect(status().isOk()); @@ -298,12 +258,9 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testAcknowledgeAlarmViaCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); + Mockito.reset(tbClusterService, auditLogService); doPost("/api/alarm/" + alarm.getId() + "/ack").andExpect(status().isOk()); @@ -320,13 +277,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); - loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -341,13 +293,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testClearAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); - loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); @@ -362,13 +309,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testAcknowledgeAlarmViaDifferentCustomer() throws Exception { loginCustomerUser(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ADDED); - loginDifferentCustomer(); Mockito.reset(tbClusterService, auditLogService); @@ -383,13 +325,8 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest { public void testAcknowledgeAlarmViaDifferentTenant() throws Exception { loginTenantAdmin(); - Mockito.reset(tbClusterService, auditLogService); - Alarm alarm = createAlarm(TEST_ALARM_TYPE); - testNotifyEntityOne(alarm, alarm.getId(), alarm.getOriginator(), - tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED); - loginDifferentTenant(); Mockito.reset(tbClusterService, auditLogService); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index 0aa81c2d18..2ad239f354 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -93,14 +93,16 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest Mockito.reset(tbClusterService, auditLogService); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + + testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), + savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED); + Assert.assertNotNull(savedCustomer); Assert.assertNotNull(savedCustomer.getId()); Assert.assertTrue(savedCustomer.getCreatedTime() > 0); Assert.assertEquals(customer.getTitle(), savedCustomer.getTitle()); - testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), - tenantAdmin.getEmail(), ActionType.ADDED); savedCustomer.setTitle("My new customer"); @@ -108,8 +110,9 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doPost("/api/customer", savedCustomer, Customer.class); - testNotifyEntityOne(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); + testNotifyEntityOne(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(), + savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UPDATED); Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); Assert.assertEquals(foundCustomer.getTitle(), savedCustomer.getTitle()); @@ -117,89 +120,65 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); - testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, savedCustomer.getId().getId().toString()); + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), + savedCustomer.getId(), savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), + tenantAdmin.getEmail(), ActionType.DELETED, savedCustomer.getId().getId().toString()); } @Test public void testSaveCustomerWithViolationOfValidation() throws Exception { Customer customer = new Customer(); + customer.setTitle(RandomStringUtils.randomAlphabetic(300)); Mockito.reset(tbClusterService, auditLogService); - doPost("/api/customer", customer) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Customer title should be specified"))); - - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: empty title")); - - customer.setTitle(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad title")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); customer.setTitle("Normal title"); - customer.setEmail("invalid@mail"); - doPost("/api/customer", customer) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Invalid email address format 'invalid@mail'"))); - - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: invalid email")); - - customer.setEmail("normal@mail.com.ua"); - customer.setCity(RandomStringUtils.randomAlphabetic(300)); + doPost("/api/customer", customer).andExpect(statusReason(containsString("length of city must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad City")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); customer.setCity("Normal city"); customer.setCountry(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of country must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Country")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); customer.setCountry("Ukraine"); customer.setPhone(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of phone must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Phone")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); customer.setPhone("+3892555554512"); customer.setState(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of state must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad state")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); customer.setState("Normal state"); customer.setZip(RandomStringUtils.randomAlphabetic(300)); doPost("/api/customer", customer).andExpect(statusReason(containsString("length of zip or postal code must be equal or less than 255"))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("Test: bad Zip")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); } @Test public void testUpdateCustomerFromDifferentTenant() throws Exception { Customer customer = new Customer(); customer.setTitle("My customer"); - - Mockito.reset(tbClusterService, auditLogService); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - - testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED); - doPost("/api/customer", savedCustomer, Customer.class); loginDifferentTenant(); @@ -211,7 +190,6 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest testNotifyEntityNever(savedCustomer.getId(), savedCustomer); deleteDifferentTenant(); - login(tenantAdmin.getName(), "testPassword1"); Mockito.reset(tbClusterService, auditLogService); @@ -228,25 +206,20 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest public void testFindCustomerById() throws Exception { Customer customer = new Customer(); customer.setTitle("My customer"); - - Mockito.reset(tbClusterService, auditLogService); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - testNotifyEntityOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), - tenantAdmin.getEmail(), ActionType.ADDED); - Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); Assert.assertNotNull(foundCustomer); Assert.assertEquals(savedCustomer, foundCustomer); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); - testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, savedCustomer.getId().getId().toString()); + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), + savedCustomer.getId(), savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), + tenantAdmin.getEmail(), ActionType.DELETED, savedCustomer.getId().getId().toString()); } @Test @@ -260,14 +233,38 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); - testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), - savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, savedCustomer.getId().getId().toString()); + testNotifyEntityBroadcastEntityStateChangeEventOneMsgToEdgeServiceNever(savedCustomer, savedCustomer.getId(), + savedCustomer.getId(), savedCustomer.getTenantId(), savedCustomer.getId(), tenantAdmin.getId(), + tenantAdmin.getEmail(), ActionType.DELETED, savedCustomer.getId().getId().toString()); doGet("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isNotFound()); } + @Test + public void testSaveCustomerWithEmptyTitle() throws Exception { + Customer customer = new Customer(); + doPost("/api/customer", customer) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Customer title should be specified"))); + } + + @Test + public void testSaveCustomerWithInvalidEmail() throws Exception { + Customer customer = new Customer(); + customer.setTitle("My customer"); + customer.setEmail("invalid@mail"); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/customer", customer) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Invalid email address format 'invalid@mail'"))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + } + @Test public void testFindCustomers() throws Exception { TenantId tenantId = savedTenant.getId(); From e3ec5db618605839a22ca6a394b861465756106c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 14 Jun 2022 14:52:59 +0300 Subject: [PATCH 218/262] Set external ids of referenced entities on export --- .../DefaultExportableEntitiesService.java | 33 +++++++++++++++---- .../exporting/ExportableEntitiesService.java | 2 ++ .../ie/exporting/impl/AssetExportService.java | 30 +++++++++++++++++ .../impl/BaseEntityExportService.java | 11 ++++++- .../impl/DefaultEntityExportService.java | 2 +- .../server/dao/ExportableEntityDao.java | 5 ++- .../server/dao/asset/AssetDao.java | 3 +- .../server/dao/customer/CustomerDao.java | 3 +- .../server/dao/dashboard/DashboardDao.java | 3 +- .../server/dao/device/DeviceDao.java | 3 +- .../server/dao/device/DeviceProfileDao.java | 3 +- .../server/dao/entityview/EntityViewDao.java | 3 +- .../server/dao/rule/RuleChainDao.java | 3 +- .../server/dao/sql/asset/AssetRepository.java | 4 +++ .../server/dao/sql/asset/JpaAssetDao.java | 7 ++++ .../dao/sql/customer/CustomerRepository.java | 4 +++ .../dao/sql/customer/JpaCustomerDao.java | 7 ++++ .../sql/dashboard/DashboardRepository.java | 5 +++ .../dao/sql/dashboard/JpaDashboardDao.java | 8 +++++ .../sql/device/DeviceProfileRepository.java | 4 +++ .../dao/sql/device/DeviceRepository.java | 4 +++ .../server/dao/sql/device/JpaDeviceDao.java | 7 ++++ .../dao/sql/device/JpaDeviceProfileDao.java | 8 +++++ .../sql/entityview/EntityViewRepository.java | 3 ++ .../dao/sql/entityview/JpaEntityViewDao.java | 7 ++++ .../server/dao/sql/rule/JpaRuleChainDao.java | 8 +++++ .../dao/sql/rule/RuleChainRepository.java | 3 ++ .../dao/sql/widget/JpaWidgetsBundleDao.java | 19 ++++++++--- .../sql/widget/WidgetsBundleRepository.java | 3 ++ .../server/dao/widget/WidgetsBundleDao.java | 2 +- 30 files changed, 184 insertions(+), 23 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index fbc4dd4bc2..822ce9025c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -75,7 +75,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi E entity = null; if (dao instanceof ExportableEntityDao) { - ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); } if (entity == null || !belongsToTenant(entity, tenantId)) { @@ -113,7 +113,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi E entity = null; if (dao instanceof ExportableEntityDao) { - ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; try { entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); } catch (UnsupportedOperationException ignored) { @@ -128,12 +128,22 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi @Override public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { - Dao dao = getDao(entityType); - if (dao instanceof ExportableEntityDao) { - ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; - return exportableEntityDao.findByTenantId(tenantId.getId(), pageLink); + ExportableEntityDao dao = getExportableEntityDao(entityType); + if (dao != null) { + return dao.findByTenantId(tenantId.getId(), pageLink); + } else { + return new PageData<>(); + } + } + + @Override + public I getExternalIdByInternal(I internalId) { + ExportableEntityDao dao = getExportableEntityDao(internalId.getEntityType()); + if (dao != null) { + return dao.getExternalIdByInternal(internalId); + } else { + return null; } - return new PageData<>(); } private boolean belongsToTenant(HasId entity, TenantId tenantId) { @@ -151,6 +161,15 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi entityRemover.accept(tenantId, id); } + private > ExportableEntityDao getExportableEntityDao(EntityType entityType) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + return (ExportableEntityDao) dao; + } else { + return null; + } + } + @SuppressWarnings("unchecked") private Dao getDao(EntityType entityType) { return (Dao) daos.get(entityType); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 493ac0b9f2..c247c51efa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -38,6 +38,8 @@ public interface ExportableEntitiesService { , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + I getExternalIdByInternal(I internalId); + void removeById(TenantId tenantId, I id); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java new file mode 100644 index 0000000000..91b637be0f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -0,0 +1,30 @@ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(TenantId tenantId, Asset asset, EntityExportData exportData, EntityExportSettings settings) { + asset.setCustomerId(getExternalIdOrElseInternal(asset.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ASSET); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 9e14531e48..952725f7ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import java.util.Optional; import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @@ -36,7 +37,15 @@ public abstract class BaseEntityExportService ID getExternalIdOrElseInternal(ID internalId) { + if (internalId == null || internalId.isNullUid()) return internalId; + return Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) + .orElse(internalId); + } + + protected D newExportData() { + return (D) new EntityExportData(); + }; public abstract Set getSupportedEntityTypes(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 1c1d70b0ae..26a076d57a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -52,7 +52,7 @@ import java.util.stream.Collectors; public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { @Autowired @Lazy - private ExportableEntitiesService exportableEntitiesService; + protected ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index aaa35e109a..77800c31e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -16,12 +16,13 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import java.util.UUID; -public interface ExportableEntityDao> extends Dao { +public interface ExportableEntityDao> extends Dao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); @@ -29,4 +30,6 @@ public interface ExportableEntityDao> extends Dao< PageData findByTenantId(UUID tenantId, PageLink pageLink); + I getExternalIdByInternal(I internalId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 495886bcc4..e3c51b48a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -34,7 +35,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index 89672d2a73..0b2abcf517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.customer; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 87804fab80..4246137d9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.dashboard; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; @@ -27,7 +28,7 @@ import java.util.UUID; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 90f2b218a7..c96f196d44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -36,7 +37,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index b79cbb05ed..4d4c73728f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -25,7 +26,7 @@ import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; -public interface DeviceProfileDao extends Dao, ExportableEntityDao { +public interface DeviceProfileDao extends Dao, ExportableEntityDao { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, UUID deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index 49d1b94ecc..75a9c122e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -32,7 +33,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/28/2017. */ -public interface EntityViewDao extends Dao, ExportableEntityDao { +public interface EntityViewDao extends Dao, ExportableEntityDao { /** * Find entity view info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 1d6c1bf29a..d19074aa53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -30,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 140ae3af86..6d4667d8ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -144,4 +144,8 @@ public interface AssetRepository extends JpaRepository, Expor Pageable pageable); Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); + + @Query("SELECT externalId FROM AssetEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 8879809ddb..b18d6c5f51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -224,6 +225,12 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return findAssetsByTenantId(tenantId, pageLink); } + @Override + public AssetId getExternalIdByInternal(AssetId internalId) { + return Optional.ofNullable(assetRepository.getExternalIdById(internalId.getId())) + .map(AssetId::new).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 1300fcdfa5..b2134811c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -39,4 +39,8 @@ public interface CustomerRepository extends JpaRepository, CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); Long countByTenantId(UUID tenantId); + + @Query("SELECT externalId FROM CustomerEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 034431157a..ab3a4b4f8d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -20,6 +20,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -85,6 +86,12 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, Pageable pageable); + @Query("SELECT externalId FROM DashboardEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 8fa0cbdc50..425c54c647 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -20,6 +20,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,6 +30,7 @@ import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.List; +import java.util.Optional; import java.util.UUID; /** @@ -65,6 +67,12 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantIdAndTitle(UUID tenantId, String title) { return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 643e5f9507..fea0e4bfc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -67,4 +67,8 @@ public interface DeviceProfileRepository extends JpaRepository, Exp "INNER JOIN DeviceProfileEntity p ON d.deviceProfileId = p.id " + "WHERE p.transportType = :transportType") Page findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType, Pageable pageable); + + @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index d4663f60a4..8513e440b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUtil; @@ -318,6 +319,12 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return findDevicesByTenantId(tenantId, pageLink); } + @Override + public DeviceId getExternalIdByInternal(DeviceId internalId) { + return Optional.ofNullable(deviceRepository.getExternalIdById(internalId.getId())) + .map(DeviceId::new).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 4cab209f49..07a680493f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -33,6 +34,7 @@ import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Objects; +import java.util.Optional; import java.util.UUID; @Component @@ -126,6 +128,12 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + @Query("SELECT externalId FROM RuleChainEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 5f3445f038..1ae399e28d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetsBundle; @@ -29,6 +31,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -88,11 +91,6 @@ public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao, ExportableEntityDao { +public interface WidgetsBundleDao extends Dao, ExportableEntityDao { /** * Save or update widgets bundle object From 7c16904c5ec6d377ed18aa5025a64ac2c976573c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Jun 2022 15:13:27 +0300 Subject: [PATCH 219/262] Send events after transaction finish --- .../DefaultEntitiesExportImportService.java | 20 ++-- .../sync/ie/EntitiesExportImportService.java | 5 +- .../impl/BaseEntityImportService.java | 8 +- .../DefaultEntitiesVersionControlService.java | 103 ++++++++---------- .../sync/vc/data/EntitiesImportCtx.java | 40 ++++++- .../data/sync/vc/EntityTypeLoadResult.java | 4 + 6 files changed, 100 insertions(+), 80 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 57404a958e..4794a659ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; @@ -82,8 +83,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } @Override - public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, - boolean saveReferencesAndSendEvents) throws ThingsboardException { + public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException { if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } @@ -95,19 +95,19 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityImportService> importService = getImportService(entityType); EntityImportResult importResult = importService.importEntity(ctx, exportData); - - if (saveReferencesAndSendEvents) { - importResult.getSaveReferencesCallback().run(); - importResult.getSendEventsCallback().run(); - saveRelations(ctx); - } - ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); + + ctx.addReferenceCallback(importResult.getSaveReferencesCallback()); + ctx.addEventCallback(importResult.getSendEventsCallback()); return importResult; } @Override - public void saveRelations(EntitiesImportCtx ctx) throws ThingsboardException { + public void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException { + for (ThrowingRunnable saveReferencesCallback : ctx.getReferenceCallbacks()) { + saveReferencesCallback.run(); + } + relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); for (EntityRelation relation : ctx.getRelations()) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index f0f41c7cdf..05e554f114 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -32,11 +32,10 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData, - boolean saveReferencesAndSendEvents) throws ThingsboardException; + , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; - void saveRelations(EntitiesImportCtx ctx) throws ThingsboardException; + void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; Comparator getEntityTypeComparatorForImport(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 2c1fd77248..df6b770d1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -57,11 +57,9 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -171,10 +169,8 @@ public abstract class BaseEntityImportService future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return Futures.transform(future, entityData -> doInTemplate(status -> loadSingleEntity(user, config, entityData)), executor); + return Futures.transform(future, entityData -> doInTemplate(user, ctx -> loadSingleEntity(ctx, config, entityData)), executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return executor.submit(() -> doInTemplate(status -> loadMultipleEntities(user, versionLoadRequest))); + return executor.submit(() -> doInTemplate(user, ctx -> loadMultipleEntities(ctx, versionLoadRequest))); } default: throw new IllegalArgumentException("Unsupported version load request"); } } - private VersionLoadResult doInTemplate(TransactionCallback result) { + private VersionLoadResult doInTemplate(SecurityUser user, Function function) { try { - return transactionTemplate.execute(result); + EntitiesImportCtx ctx = new EntitiesImportCtx(user); + VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx)); + try { + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + return result; } catch (LoadEntityException e) { return onError(e.getData(), e.getCause()); } } - private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) { + private VersionLoadResult loadSingleEntity(EntitiesImportCtx ctx, VersionLoadConfig config, EntityExportData entityData) { try { - var ctx = new EntitiesImportCtx(user, EntityImportSettings.builder() + ctx.setSettings(EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) .findExistingByName(false) .build()); - EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, true); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + exportImportService.saveReferencesAndRelations(ctx); return VersionLoadResult.success(EntityTypeLoadResult.builder() .entityType(importResult.getEntityType()) @@ -268,29 +282,19 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private VersionLoadResult loadMultipleEntities(SecurityUser user, EntityTypeVersionLoadRequest request) { - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - Map toReimport = new HashMap<>(); - List saveReferencesCallbacks = new ArrayList<>(); - List sendEventsCallbacks = new ArrayList<>(); - - EntitiesImportCtx ctx = new EntitiesImportCtx(user); + private VersionLoadResult loadMultipleEntities(EntitiesImportCtx ctx, EntityTypeVersionLoadRequest request) { var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); for (EntityType entityType : entityTypes) { EntityTypeVersionLoadConfig config = request.getEntityTypes().get(entityType); - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); - int limit = 100; int offset = 0; List entityDataList; do { try { - entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); + entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } @@ -304,46 +308,33 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { - importResult = exportImportService.importEntity(ctx, entityData, false); + importResult = exportImportService.importEntity(ctx, entityData); } catch (Exception e) { throw new LoadEntityException(entityData, e); } if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - toReimport.put(entityData.getEntity().getExternalId(), ctx.getSettings()); + ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings()); continue; } - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - - importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + ctx.registerResult(entityType, importResult.getOldEntity() == null); + ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } offset += limit; } while (entityDataList.size() == limit); - results.put(entityType, EntityTypeLoadResult.builder() - .entityType(entityType) - .created(created.get()) - .updated(updated.get()) - .build()); } sw.startNew("Reimport"); - toReimport.forEach((externalId, importSettings) -> { + ctx.getToReimport().forEach((externalId, importSettings) -> { try { - EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get(); + EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), request.getVersionId(), externalId).get(); importSettings.setResetExternalIdsOfAnotherTenant(true); ctx.setSettings(importSettings); - EntityImportResult importResult = exportImportService.importEntity(ctx, entityData, false); - - EntityTypeLoadResult stats = results.get(externalId.getEntityType()); - if (importResult.getOldEntity() == null) stats.setCreated(stats.getCreated() + 1); - else stats.setUpdated(stats.getUpdated() + 1); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - importedEntities.computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + ctx.registerResult(externalId.getEntityType(), importResult.getOldEntity() == null); + ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } catch (Exception e) { throw new RuntimeException(e); @@ -356,17 +347,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) .forEach(entityType -> { DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); }, 100, entity -> { - if (importedEntities.get(entityType) == null || !importedEntities.get(entityType).contains(entity.getId())) { - exportableEntitiesService.removeById(user.getTenantId(), entity.getId()); + if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { + exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); - sendEventsCallbacks.add(() -> { - entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), - entity, null, ActionType.DELETED, null, user); + ctx.addEventCallback(() -> { + entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, ctx.getUser()); }); - EntityTypeLoadResult result = results.get(entityType); - result.setDeleted(result.getDeleted() + 1); + ctx.registerDeleted(entityType); } }); }); @@ -374,14 +364,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont sw.startNew("Callbacks"); try { - for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { - saveReferencesCallback.run(); - } - for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { - sendEventsCallback.run(); - } - sw.startNew("Relations"); - exportImportService.saveRelations(ctx); + exportImportService.saveReferencesAndRelations(ctx); } catch (ThingsboardException e) { throw new RuntimeException(e); } @@ -391,7 +374,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); } log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); - return VersionLoadResult.success(new ArrayList<>(results.values())); + return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())); } private VersionLoadResult onError(EntityExportData entityData, Throwable e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index ec7c9c4fdf..f37e49ceed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -17,16 +17,20 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -37,7 +41,14 @@ public class EntitiesImportCtx { private final SecurityUser user; private EntityImportSettings settings; + Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); + Map toReimport = new HashMap<>(); + private final List referenceCallbacks = new ArrayList<>(); + private final List eventCallbacks = new ArrayList<>(); + private final Map externalToInternalIdMap = new HashMap<>(); + private final Set relations = new LinkedHashSet<>(); public EntitiesImportCtx(SecurityUser user) { @@ -84,8 +95,35 @@ public class EntitiesImportCtx { externalToInternalIdMap.put(externalId, internalId); } + public void registerResult(EntityType entityType, boolean created) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + if (created) { + result.setCreated(result.getCreated() + 1); + } else { + result.setUpdated(result.getUpdated() + 1); + } + } + + public void registerDeleted(EntityType entityType) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + result.setDeleted(result.getDeleted() + 1); + } + public void addRelations(Collection values) { relations.addAll(values); } + public void addReferenceCallback(ThrowingRunnable tr) { + if (tr != null) { + referenceCallbacks.add(tr); + } + } + + public void addEventCallback(ThrowingRunnable tr) { + if (tr != null) { + eventCallbacks.add(tr); + } + } + + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java index cc0a8a642a..84a28d0770 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -30,4 +30,8 @@ public class EntityTypeLoadResult { private int created; private int updated; private int deleted; + + public EntityTypeLoadResult(EntityType entityType) { + this.entityType = entityType; + } } From 5ec1793f6ee1e791e461859cfc442e6dc85ad473 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Jun 2022 16:05:27 +0300 Subject: [PATCH 220/262] To simplify merge --- .../DefaultEntitiesVersionControlService.java | 171 +++++++++--------- .../sync/vc/data/EntitiesImportCtx.java | 8 +- 2 files changed, 93 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d26a6e5ecb..6b299930d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -20,12 +20,11 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.TbStopWatch; @@ -33,7 +32,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -43,7 +41,6 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -72,11 +69,9 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; -import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; @@ -90,16 +85,11 @@ import javax.annotation.PreDestroy; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -231,20 +221,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return Futures.transform(future, entityData -> doInTemplate(user, ctx -> loadSingleEntity(ctx, config, entityData)), executor); + return Futures.transform(future, entityData -> doInTemplate(user, request, ctx -> loadSingleEntity(ctx, config, entityData)), executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return executor.submit(() -> doInTemplate(user, ctx -> loadMultipleEntities(ctx, versionLoadRequest))); + return executor.submit(() -> doInTemplate(user, request, ctx -> loadMultipleEntities(ctx, versionLoadRequest))); } default: throw new IllegalArgumentException("Unsupported version load request"); } } - private VersionLoadResult doInTemplate(SecurityUser user, Function function) { + private VersionLoadResult doInTemplate(SecurityUser user, VersionLoadRequest request, Function function) { try { - EntitiesImportCtx ctx = new EntitiesImportCtx(user); + EntitiesImportCtx ctx = new EntitiesImportCtx(user, request.getVersionId()); VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx)); try { for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { @@ -282,53 +272,87 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } + @SneakyThrows private VersionLoadResult loadMultipleEntities(EntitiesImportCtx ctx, EntityTypeVersionLoadRequest request) { var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); for (EntityType entityType : entityTypes) { - EntityTypeVersionLoadConfig config = request.getEntityTypes().get(entityType); - int limit = 100; - int offset = 0; - List entityDataList; - do { + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + sw.startNew("Entities " + entityType.name()); + EntityImportSettings settings = getEntityImportSettings(request, entityType); + ctx.setSettings(settings); + importEntities(ctx, entityType); + } + + sw.startNew("Reimport"); + reimport(ctx); + + sw.startNew("Remove Others"); + request.getEntityTypes().keySet().stream() + .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> removeOtherEntities(ctx, entityType)); + + sw.startNew("References and Relations"); + + exportImportService.saveReferencesAndRelations(ctx); + + sw.stop(); + for (var task : sw.getTaskInfo()) { + log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); + } + log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); + return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())); + } + + private EntityImportSettings getEntityImportSettings(EntityTypeVersionLoadRequest request, EntityType entityType) { + var config = request.getEntityTypes().get(entityType); + return EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void importEntities(EntitiesImportCtx ctx, EntityType entityType) { + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + EntityImportResult importResult; try { - entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); + importResult = exportImportService.importEntity(ctx, entityData); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); } - ctx.setSettings(EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(config.isFindExistingEntityByName()) - .build()); - for (EntityExportData entityData : entityDataList) { - sw.startNew("Entities " + entityType.name()); - log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); - EntityImportResult importResult; - try { - importResult = exportImportService.importEntity(ctx, entityData); - } catch (Exception e) { - throw new LoadEntityException(entityData, e); - } - if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings()); - continue; - } - - ctx.registerResult(entityType, importResult.getOldEntity() == null); - ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) - .add(importResult.getSavedEntity().getId()); + if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { + ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings()); + continue; } - offset += limit; - } while (entityDataList.size() == limit); - } - sw.startNew("Reimport"); + ctx.registerResult(entityType, importResult.getOldEntity() == null); + ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) + .add(importResult.getSavedEntity().getId()); + } + offset += limit; + } while (entityDataList.size() == limit); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void reimport(EntitiesImportCtx ctx) { ctx.getToReimport().forEach((externalId, importSettings) -> { try { - EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), request.getVersionId(), externalId).get(); + EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), ctx.getVersionId(), externalId).get(); importSettings.setResetExternalIdsOfAnotherTenant(true); ctx.setSettings(importSettings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); @@ -340,41 +364,22 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException(e); } }); + } - sw.startNew("Remove Others"); - request.getEntityTypes().keySet().stream() - .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) - .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) - .forEach(entityType -> { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { - exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); - - ctx.addEventCallback(() -> { - entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(), - entity, null, ActionType.DELETED, null, ctx.getUser()); - }); - ctx.registerDeleted(entityType); - } - }); - }); - - sw.startNew("Callbacks"); - - try { - exportImportService.saveReferencesAndRelations(ctx); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } + private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { + exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); - sw.stop(); - for (var task : sw.getTaskInfo()) { - log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); - } - log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); - return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())); + ctx.addEventCallback(() -> { + entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, ctx.getUser()); + }); + ctx.registerDeleted(entityType); + } + }); } private VersionLoadResult onError(EntityExportData entityData, Throwable e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index f37e49ceed..94693009b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -39,6 +39,7 @@ import java.util.Set; public class EntitiesImportCtx { private final SecurityUser user; + private final String versionId; private EntityImportSettings settings; Map results = new HashMap<>(); @@ -51,12 +52,13 @@ public class EntitiesImportCtx { private final Set relations = new LinkedHashSet<>(); - public EntitiesImportCtx(SecurityUser user) { - this(user, null); + public EntitiesImportCtx(SecurityUser user, String versionId) { + this(user, versionId, null); } - public EntitiesImportCtx(SecurityUser user, EntityImportSettings settings) { + public EntitiesImportCtx(SecurityUser user, String versionId, EntityImportSettings settings) { this.user = user; + this.versionId = versionId; this.settings = settings; } From 51e05d1a9ad7bc60d824949d57639c56dc4d0666 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Tue, 14 Jun 2022 16:14:01 +0300 Subject: [PATCH 221/262] UI: Refactoring phone input and add to test sms dialog and contact --- .../admin/send-test-sms-dialog.component.html | 16 ++++------ .../sms-auth-dialog.component.html | 2 +- .../shared/components/contact.component.html | 12 ++++---- .../components/phone-input.component.html | 2 +- .../components/phone-input.component.scss | 1 - .../components/phone-input.component.ts | 30 ++++++++++++++----- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/send-test-sms-dialog.component.html b/ui-ngx/src/app/modules/home/pages/admin/send-test-sms-dialog.component.html index cc7b4d21f3..40cc60f730 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/send-test-sms-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/send-test-sms-dialog.component.html @@ -30,17 +30,11 @@
- - admin.number-to - - - {{ 'admin.number-to-required' | translate }} - - - {{ 'admin.phone-number-pattern' | translate }} - - - + + admin.sms-message diff --git a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html index e216c22d3e..aace1d7152 100644 --- a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/sms-auth-dialog.component.html @@ -36,7 +36,7 @@ {{ 'security.2fa.dialog.sms-step-label' | translate }}

security.2fa.dialog.sms-step-description

-
+
contact.address2 - - contact.phone - - - {{ 'contact.phone-max-length' | translate }} - - + + contact.email diff --git a/ui-ngx/src/app/shared/components/phone-input.component.html b/ui-ngx/src/app/shared/components/phone-input.component.html index c5b2ae7bc1..83db450d70 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.html +++ b/ui-ngx/src/app/shared/components/phone-input.component.html @@ -28,7 +28,7 @@
- {{ placeholder | translate }} + {{ label | translate }} = this.countryCodeData.allCountries; phonePlaceholder: string; From 5df2ca5372d88820d3c5f38ce94d87203f93cfcd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 14 Jun 2022 16:42:33 +0300 Subject: [PATCH 222/262] Set external ids instead of internal on export --- .../ie/exporting/impl/AssetExportService.java | 16 ++++++- .../impl/BaseEntityExportService.java | 6 --- .../impl/DashboardExportService.java | 48 +++++++++++++++++++ .../impl/DefaultEntityExportService.java | 12 +++++ .../exporting/impl/DeviceExportService.java | 2 + .../impl/DeviceProfileExportService.java | 44 +++++++++++++++++ .../impl/EntityViewExportService.java | 44 +++++++++++++++++ .../server/common/data/Customer.java | 2 +- .../server/common/data/Dashboard.java | 2 +- 9 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index 91b637be0f..be9d123484 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; @@ -14,7 +29,6 @@ import java.util.Set; @Service @TbCoreComponent -@RequiredArgsConstructor public class AssetExportService extends BaseEntityExportService> { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 952725f7ce..dd1f6a5c49 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -37,12 +37,6 @@ public abstract class BaseEntityExportService ID getExternalIdOrElseInternal(ID internalId) { - if (internalId == null || internalId.isNullUid()) return internalId; - return Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) - .orElse(internalId); - } - protected D newExportData() { return (D) new EntityExportData(); }; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java new file mode 100644 index 0000000000..de87559787 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Set; + +@Service +@TbCoreComponent +public class DashboardExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, EntityExportSettings settings) { + if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) { + dashboard.getAssignedCustomers().forEach(customerInfo -> { + customerInfo.setCustomerId(getExternalIdOrElseInternal(customerInfo.getCustomerId())); + }); + } + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DASHBOARD); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 26a076d57a..16838e1cdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -43,6 +43,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -67,6 +68,7 @@ public class DefaultEntityExportService relations = exportRelations(user, entity); + relations.forEach(relation -> { + relation.setFrom(getExternalIdOrElseInternal(relation.getFrom())); + relation.setTo(getExternalIdOrElseInternal(relation.getTo())); + }); exportData.setRelations(relations); } if (exportSettings.isExportAttributes()) { @@ -126,6 +132,12 @@ public class DefaultEntityExportService ID getExternalIdOrElseInternal(ID internalId) { + if (internalId == null || internalId.isNullUid()) return internalId; + return Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) + .orElse(internalId); + } + protected D newExportData() { return (D) new EntityExportData(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 0aa8933a27..efb6fe8c96 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -37,6 +37,8 @@ public class DeviceExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, EntityExportSettings settings) { + deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(deviceProfile.getDefaultDashboardId())); + deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(deviceProfile.getDefaultRuleChainId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE_PROFILE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java new file mode 100644 index 0000000000..10606f076f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Set; + +@Service +@TbCoreComponent +public class EntityViewExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(TenantId tenantId, EntityView entityView, EntityExportData exportData, EntityExportSettings settings) { + entityView.setEntityId(getExternalIdOrElseInternal(entityView.getEntityId())); + entityView.setCustomerId(getExternalIdOrElseInternal(entityView.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ENTITY_VIEW); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index 92e5e9ae78..9ce638e667 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 59c122f3d0..b9f2bf1fee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -22,7 +22,7 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; From c9558b5e08916882ba41aabb0bc23d67548a68f0 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Jun 2022 16:59:21 +0300 Subject: [PATCH 223/262] refactoring: commit2 (error) --- .../controller/AbstractNotifyEntityTest.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 40a3bac5de..38139d91d4 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.model.ModelConstants; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Locale; import static org.mockito.Mockito.never; @@ -96,12 +98,14 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), - Mockito.any(exp.getClass()), Mockito.eq(additionalInfo)); + Mockito.argThat(argument -> + argument.getMessage().equals(exp.getMessage())), Mockito.eq(additionalInfo)); } else { Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), - Mockito.any(exp.getClass())); + Mockito.argThat(argument -> + argument.getMessage().equals(exp.getMessage()))); } Mockito.verify(tbClusterService, never()).pushMsgToRuleEngine(Mockito.any(), Mockito.any(entity_NULL_UUID.getClass()), Mockito.any(), Mockito.any()); @@ -170,4 +174,10 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } return result; } + + private String getFailureStack(Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + return sw.toString(); + } } From 281c266d26d0d73c6a9379f006e855ddf7adf7ef Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Jun 2022 17:00:55 +0300 Subject: [PATCH 224/262] refactoring: commit3 (error) --- .../BaseCustomerControllerTest.java | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index 2ad239f354..c3ae7d76f9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -128,50 +128,57 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest @Test public void testSaveCustomerWithViolationOfValidation() throws Exception { Customer customer = new Customer(); + String validationError = "Validation error: "; customer.setTitle(RandomStringUtils.randomAlphabetic(300)); Mockito.reset(tbClusterService, auditLogService); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); + String msgError = "length of title must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); customer.setTitle("Normal title"); customer.setCity(RandomStringUtils.randomAlphabetic(300)); - - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of city must be equal or less than 255"))); + msgError = "length of city must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); customer.setCity("Normal city"); customer.setCountry(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of country must be equal or less than 255"))); + msgError = "length of country must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); customer.setCountry("Ukraine"); customer.setPhone(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of phone must be equal or less than 255"))); + msgError = "length of phone must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); customer.setPhone("+3892555554512"); customer.setState(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of state must be equal or less than 255"))); + msgError = "length of state must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); customer.setState("Normal state"); customer.setZip(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/customer", customer).andExpect(statusReason(containsString("length of zip or postal code must be equal or less than 255"))); + msgError = "length of zip or postal code must be equal or less than 255"; + doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + } @Test @@ -244,14 +251,22 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest @Test public void testSaveCustomerWithEmptyTitle() throws Exception { Customer customer = new Customer(); + String msgError = "Customer title should be specified"; + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/customer", customer) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Customer title should be specified"))); + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError + "!")); } @Test public void testSaveCustomerWithInvalidEmail() throws Exception { Customer customer = new Customer(); + String msgError = "Invalid email address format 'invalid@mail'"; customer.setTitle("My customer"); customer.setEmail("invalid@mail"); @@ -259,10 +274,10 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doPost("/api/customer", customer) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Invalid email address format 'invalid@mail'"))); + .andExpect(statusReason(containsString(msgError))); testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException("")); + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError + "!")); } @Test From 605c32123f39fb5e4a56311d8278d95cc30fff0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Kwakernaak?= <58462885+andre-aktivconsultancy@users.noreply.github.com> Date: Thu, 23 Dec 2021 10:55:20 +0100 Subject: [PATCH 225/262] ensure postgres database is always started The Postgres database would only get started on first launch. When starting a second time e.g. after a `docker-compose down` and `docker-compose up`, the database was not started resulting in a failure when ThingsBoard is starting. (cherry picked from commit 7d5153f52ee4033c8a7ffab0a678950122b982eb) --- msa/tb/docker-postgres/start-db.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/msa/tb/docker-postgres/start-db.sh b/msa/tb/docker-postgres/start-db.sh index ff2805feab..def5165f7a 100644 --- a/msa/tb/docker-postgres/start-db.sh +++ b/msa/tb/docker-postgres/start-db.sh @@ -22,14 +22,22 @@ PG_CTL=$(find /usr/lib/postgresql/ -name pg_ctl) if [ ! -d ${PGDATA} ]; then mkdir -p ${PGDATA} ${PG_CTL} initdb +else + ${PG_CTL} start fi exec setsid nohup postgres >> ${PGLOG}/postgres.log 2>&1 & if [ ! -f ${firstlaunch} ]; then sleep 2 - while ! psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" + while ! psql -U thingsboard -d postgres -c "CREATE DATABASE thingsboard" do sleep 1 done -fi \ No newline at end of file +else + until pg_isready --dbname thingsboard --quiet + do + sleep 1 + echo "Waiting for db" + done +fi From 4e0e1d3caedb2884fbe7b0af8d9907d8847ed52d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 14 Jun 2022 19:16:38 +0300 Subject: [PATCH 226/262] Postgres startup script improved. will wait for postgres is ready about 300+ seconds. Postgres may do some recovery stuff on startup. --- msa/tb/docker-cassandra/start-db.sh | 7 +++++++ msa/tb/docker-postgres/start-db.sh | 9 ++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/msa/tb/docker-cassandra/start-db.sh b/msa/tb/docker-cassandra/start-db.sh index 92423de9a8..8f8e4a7c35 100644 --- a/msa/tb/docker-cassandra/start-db.sh +++ b/msa/tb/docker-cassandra/start-db.sh @@ -32,6 +32,13 @@ if [ ! -f ${firstlaunch} ]; then do sleep 1 done +else + RETRIES="${PG_ISREADY_RETRIES:-300}" + until pg_isready -U ${pkg.user} -d thingsboard --quiet || [ $RETRIES -eq 0 ] + do + echo "Connecting to Postgres, $((RETRIES--)) attempts left..." + sleep 1 + done fi cassandra_data_dir=${CASSANDRA_DATA} diff --git a/msa/tb/docker-postgres/start-db.sh b/msa/tb/docker-postgres/start-db.sh index def5165f7a..0962968222 100644 --- a/msa/tb/docker-postgres/start-db.sh +++ b/msa/tb/docker-postgres/start-db.sh @@ -22,22 +22,21 @@ PG_CTL=$(find /usr/lib/postgresql/ -name pg_ctl) if [ ! -d ${PGDATA} ]; then mkdir -p ${PGDATA} ${PG_CTL} initdb -else - ${PG_CTL} start fi exec setsid nohup postgres >> ${PGLOG}/postgres.log 2>&1 & if [ ! -f ${firstlaunch} ]; then sleep 2 - while ! psql -U thingsboard -d postgres -c "CREATE DATABASE thingsboard" + while ! psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" do sleep 1 done else - until pg_isready --dbname thingsboard --quiet + RETRIES="${PG_ISREADY_RETRIES:-300}" + until pg_isready -U ${pkg.user} -d thingsboard --quiet || [ $RETRIES -eq 0 ] do + echo "Connecting to Postgres, $((RETRIES--)) attempts left..." sleep 1 - echo "Waiting for db" done fi From 92818934aba5e73d23a6bfd5e2bfc64d5d1969f8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 14 Jun 2022 22:40:01 +0300 Subject: [PATCH 227/262] Postgres startup script improved, Install db script improved. First launch file writes only ofter success install. More verbosity added during Postgresql startup --- msa/tb/docker-cassandra/start-db.sh | 26 +++++++++++++------------- msa/tb/docker-postgres/start-db.sh | 26 +++++++++++++------------- msa/tb/docker/install-tb.sh | 2 ++ msa/tb/docker/start-tb.sh | 17 ++++++++++------- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/msa/tb/docker-cassandra/start-db.sh b/msa/tb/docker-cassandra/start-db.sh index 8f8e4a7c35..d42281d155 100644 --- a/msa/tb/docker-cassandra/start-db.sh +++ b/msa/tb/docker-cassandra/start-db.sh @@ -24,23 +24,23 @@ if [ ! -d ${PGDATA} ]; then ${PG_CTL} initdb fi -exec setsid nohup postgres >> ${PGLOG}/postgres.log 2>&1 & +echo "Starting Postgresql..." +${PG_CTL} start + +RETRIES="${PG_ISREADY_RETRIES:-300}" +until pg_isready -U ${pkg.user} -d postgres --quiet || [ $RETRIES -eq 0 ] +do + echo "Connecting to Postgres, $((RETRIES--)) attempts left..." + sleep 1 +done if [ ! -f ${firstlaunch} ]; then - sleep 2 - while ! psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" - do - sleep 1 - done -else - RETRIES="${PG_ISREADY_RETRIES:-300}" - until pg_isready -U ${pkg.user} -d thingsboard --quiet || [ $RETRIES -eq 0 ] - do - echo "Connecting to Postgres, $((RETRIES--)) attempts left..." - sleep 1 - done + echo "Creating database..." + psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" fi +echo "Postgresql is ready" + cassandra_data_dir=${CASSANDRA_DATA} cassandra_data_link=/var/lib/cassandra diff --git a/msa/tb/docker-postgres/start-db.sh b/msa/tb/docker-postgres/start-db.sh index 0962968222..917287f35c 100644 --- a/msa/tb/docker-postgres/start-db.sh +++ b/msa/tb/docker-postgres/start-db.sh @@ -24,19 +24,19 @@ if [ ! -d ${PGDATA} ]; then ${PG_CTL} initdb fi -exec setsid nohup postgres >> ${PGLOG}/postgres.log 2>&1 & +echo "Starting Postgresql..." +${PG_CTL} start + +RETRIES="${PG_ISREADY_RETRIES:-300}" +until pg_isready -U ${pkg.user} -d postgres --quiet || [ $RETRIES -eq 0 ] +do + echo "Connecting to Postgres, $((RETRIES--)) attempts left..." + sleep 1 +done if [ ! -f ${firstlaunch} ]; then - sleep 2 - while ! psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" - do - sleep 1 - done -else - RETRIES="${PG_ISREADY_RETRIES:-300}" - until pg_isready -U ${pkg.user} -d thingsboard --quiet || [ $RETRIES -eq 0 ] - do - echo "Connecting to Postgres, $((RETRIES--)) attempts left..." - sleep 1 - done + echo "Creating database..." + psql -U ${pkg.user} -d postgres -c "CREATE DATABASE thingsboard" fi + +echo "Postgresql is ready" \ No newline at end of file diff --git a/msa/tb/docker/install-tb.sh b/msa/tb/docker/install-tb.sh index ff9ae7f496..3395e63c84 100644 --- a/msa/tb/docker/install-tb.sh +++ b/msa/tb/docker/install-tb.sh @@ -46,6 +46,8 @@ source "${CONF_FOLDER}/${configfile}" echo "Starting ThingsBoard installation ..." +set -e + java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ -Dinstall.load_demo=${loadDemo} \ -Dspring.jpa.hibernate.ddl-auto=none \ diff --git a/msa/tb/docker/start-tb.sh b/msa/tb/docker/start-tb.sh index 5c3b17fb72..30a16413b6 100755 --- a/msa/tb/docker/start-tb.sh +++ b/msa/tb/docker/start-tb.sh @@ -25,15 +25,18 @@ firstlaunch=${DATA_FOLDER}/.firstlaunch source "${CONF_FOLDER}/${configfile}" if [ ! -f ${firstlaunch} ]; then - install-tb.sh --loadDemo - touch ${firstlaunch} + install-tb.sh --loadDemo && touch ${firstlaunch} fi -echo "Starting ThingsBoard ..." +if [ -f ${firstlaunch} ]; then + echo "Starting ThingsBoard ..." -java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardServerApplication \ - -Dspring.jpa.hibernate.ddl-auto=none \ - -Dlogging.config=${CONF_FOLDER}/logback.xml \ - org.springframework.boot.loader.PropertiesLauncher + java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardServerApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=${CONF_FOLDER}/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher +else + echo "ERROR: ThingsBoard is not installed" +fi stop-db.sh \ No newline at end of file From 8e3054c999d34e394589483e9985545ea562da74 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 15 Jun 2022 09:42:52 +0300 Subject: [PATCH 228/262] Entities Import Context --- .../DefaultEntitiesExportImportService.java | 7 ++-- .../sync/ie/EntitiesExportImportService.java | 3 +- .../ie/exporting/EntityExportService.java | 3 +- .../impl/BaseEntityExportService.java | 7 ++-- .../impl/DefaultEntityExportService.java | 23 ++++++----- .../DefaultEntitiesVersionControlService.java | 41 ++++++++++--------- .../vc/data/ComplexEntitiesExportCtx.java | 23 +++++++++++ .../sync/vc/data/EntitiesExportCtx.java | 37 +++++++++++++++++ .../sync/vc/data/EntitiesImportCtx.java | 9 ++-- .../sync/vc/data/SimpleEntitiesExportCtx.java | 17 ++++++++ 10 files changed, 127 insertions(+), 43 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 4794a659ab..094ce55cc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.ArrayList; @@ -71,15 +72,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(user, entityId, exportSettings); + return exportService.getExportData(ctx, entityId, exportSettings); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 05e554f114..9031b1a51f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -24,13 +24,14 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.Comparator; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 9b251a3dab..07f7d6d311 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -21,9 +21,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; public interface EntityExportService, D extends EntityExportData> { - D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + D getExportData(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 9e14531e48..1877672faa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -23,15 +23,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { - setRelatedEntities(user.getTenantId(), entity, (D) exportData, exportSettings); - super.setAdditionalExportData(user, entity, exportData, exportSettings); + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + setRelatedEntities(ctx.getTenantId(), entity, (D) exportData, exportSettings); + super.setAdditionalExportData(ctx, entity, exportData, exportSettings); } protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData, EntityExportSettings settings) {} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 1c1d70b0ae..ebcf89e261 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.ArrayList; import java.util.Collections; @@ -59,44 +60,44 @@ public class DefaultEntityExportService relations = exportRelations(user, entity); + List relations = exportRelations(ctx, entity); exportData.setRelations(relations); } if (exportSettings.isExportAttributes()) { - Map> attributes = exportAttributes(user, entity); + Map> attributes = exportAttributes(ctx, entity); exportData.setAttributes(attributes); } } - private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List relations = new ArrayList<>(); - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(inboundRelations); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(outboundRelations); return relations; } - private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + private Map> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List scopes; if (entity.getId().getEntityType() == EntityType.DEVICE) { scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); @@ -106,7 +107,7 @@ public class DefaultEntityExportService> attributes = new LinkedHashMap<>(); scopes.forEach(scope -> { try { - attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() .map(attribute -> { AttributeExportData attributeExportData = new AttributeExportData(); attributeExportData.setKey(attribute.getKey()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6b299930d0..551edf7706 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -77,7 +77,10 @@ import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesServic import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; @@ -136,11 +139,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { - handleSingleEntityRequest(user, commit, gitFutures, (SingleEntityVersionCreateRequest) request); + handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request)); break; } case COMPLEX: { - handleComplexRequest(user, commit, gitFutures, (ComplexVersionCreateRequest) request); + handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request)); break; } } @@ -148,21 +151,22 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, executor); } - private void handleSingleEntityRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, SingleEntityVersionCreateRequest versionCreateRequest) throws Exception { - gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception { + ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId(), ctx.getSettings())); } - private void handleComplexRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, ComplexVersionCreateRequest versionCreateRequest) { - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + private void handleComplexRequest(ComplexEntitiesExportCtx ctx) { + ctx.getRequest().getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), ctx.getRequest().getSyncStrategy()) == SyncStrategy.OVERWRITE) { + ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType)); } + EntityExportSettings settings = ctx.getSettings(entityType); if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink) , 100, entity -> { try { - gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + ctx.add(saveEntityData(ctx, entity.getId(), settings)); } catch (Exception e) { throw new RuntimeException(e); } @@ -170,7 +174,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId), settings)); } catch (Exception e) { throw new RuntimeException(e); } @@ -179,13 +183,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); } - private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() - .exportRelations(config.isSaveRelations()) - .exportAttributes(config.isSaveAttributes()) - .exportCredentials(config.isSaveCredentials()) - .build()); - return gitServiceQueue.addToCommit(commit, entityData); + private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId, EntityExportSettings settings) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId, settings); + return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); } @Override @@ -410,7 +410,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> { - EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null); + EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId, EntityExportSettings.builder() .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java new file mode 100644 index 0000000000..673590d7f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java @@ -0,0 +1,23 @@ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.HashMap; +import java.util.Map; + +public class ComplexEntitiesExportCtx extends EntitiesExportCtx { + + private final Map settings = new HashMap<>(); + + public ComplexEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, ComplexVersionCreateRequest request) { + super(user, commit, request); + request.getEntityTypes().forEach((type, config) -> settings.put(type, buildExportSettings(config))); + } + + public EntityExportSettings getSettings(EntityType entityType) { + return settings.get(entityType); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java new file mode 100644 index 0000000000..0fa3261841 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class EntitiesExportCtx { + + protected final SecurityUser user; + protected final CommitGitRequest commit; + protected final R request; + private final List> futures = new ArrayList<>(); + + public void add(ListenableFuture future) { + futures.add(future); + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + protected static EntityExportSettings buildExportSettings(VersionCreateConfig config) { + return EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 94693009b4..497d2b9fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -40,11 +40,10 @@ public class EntitiesImportCtx { private final SecurityUser user; private final String versionId; - private EntityImportSettings settings; - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - Map toReimport = new HashMap<>(); + private final Map results = new HashMap<>(); + private final Map> importedEntities = new HashMap<>(); + private final Map toReimport = new HashMap<>(); private final List referenceCallbacks = new ArrayList<>(); private final List eventCallbacks = new ArrayList<>(); @@ -52,6 +51,8 @@ public class EntitiesImportCtx { private final Set relations = new LinkedHashSet<>(); + private EntityImportSettings settings; + public EntitiesImportCtx(SecurityUser user, String versionId) { this(user, versionId, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java new file mode 100644 index 0000000000..600d70106b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -0,0 +1,17 @@ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +public class SimpleEntitiesExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityExportSettings settings; + + public SimpleEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, SingleEntityVersionCreateRequest request) { + super(user, commit, request); + this.settings = request != null ? buildExportSettings(request.getConfig()) : null; + } +} From b55dbdf050a7f32ddb97fa59380533242e35b71e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 15 Jun 2022 10:32:19 +0300 Subject: [PATCH 229/262] Entity Export refactoring --- .../DefaultEntitiesExportImportService.java | 2 +- .../sync/ie/EntitiesExportImportService.java | 2 +- .../ie/exporting/EntityExportService.java | 2 +- .../impl/BaseEntityExportService.java | 2 +- .../impl/DefaultEntityExportService.java | 8 ++-- .../DefaultEntitiesVersionControlService.java | 26 +++++------ .../vc/data/ComplexEntitiesExportCtx.java | 20 ++++++++ .../sync/vc/data/EntitiesExportCtx.java | 35 +++++++++++++- .../sync/vc/data/EntityTypeExportCtx.java | 46 +++++++++++++++++++ .../sync/vc/data/SimpleEntitiesExportCtx.java | 16 +++++++ 10 files changed, 135 insertions(+), 24 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 094ce55cc7..2fdf5bc6c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -72,7 +72,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 9031b1a51f..44f0d84a3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -31,7 +31,7 @@ import java.util.Comparator; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 07f7d6d311..4174951e63 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -25,6 +25,6 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; public interface EntityExportService, D extends EntityExportData> { - D getExportData(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + D getExportData(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 1877672faa..f6f72d6ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -30,7 +30,7 @@ import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { setRelatedEntities(ctx.getTenantId(), entity, (D) exportData, exportSettings); super.setAdditionalExportData(ctx, entity, exportData, exportSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index ebcf89e261..8dfdbb26eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -60,7 +60,7 @@ public class DefaultEntityExportService ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { D exportData = newExportData(); E entity = exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entityId); @@ -75,7 +75,7 @@ public class DefaultEntityExportService ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { if (exportSettings.isExportRelations()) { List relations = exportRelations(ctx, entity); exportData.setRelations(relations); @@ -86,7 +86,7 @@ public class DefaultEntityExportService exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List relations = new ArrayList<>(); List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); @@ -97,7 +97,7 @@ public class DefaultEntityExportService> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + private Map> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List scopes; if (entity.getId().getEntityType() == EntityType.DEVICE) { scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 551edf7706..6dd7c87dae 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,7 +22,6 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; @@ -60,9 +59,7 @@ import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionC import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; @@ -76,10 +73,10 @@ import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; -import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx; import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -152,21 +149,22 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception { - ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId(), ctx.getSettings())); + ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId())); } - private void handleComplexRequest(ComplexEntitiesExportCtx ctx) { - ctx.getRequest().getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), ctx.getRequest().getSyncStrategy()) == SyncStrategy.OVERWRITE) { + private void handleComplexRequest(ComplexEntitiesExportCtx parentCtx) { + ComplexVersionCreateRequest request = parentCtx.getRequest(); + request.getEntityTypes().forEach((entityType, config) -> { + EntityTypeExportCtx ctx = new EntityTypeExportCtx(parentCtx, config, request.getSyncStrategy(), entityType); + if (ctx.isOverwrite()) { ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType)); } - EntityExportSettings settings = ctx.getSettings(entityType); if (config.isAllEntities()) { DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink) , 100, entity -> { try { - ctx.add(saveEntityData(ctx, entity.getId(), settings)); + ctx.add(saveEntityData(ctx, entity.getId())); } catch (Exception e) { throw new RuntimeException(e); } @@ -174,7 +172,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId), settings)); + ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId))); } catch (Exception e) { throw new RuntimeException(e); } @@ -183,8 +181,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); } - private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId, EntityExportSettings settings) throws Exception { - EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId, settings); + private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId, ctx.getSettings()); return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java index 673590d7f7..77482fdf71 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.EntityType; @@ -20,4 +35,9 @@ public class ComplexEntitiesExportCtx extends EntitiesExportCtx { +public abstract class EntitiesExportCtx { protected final SecurityUser user; protected final CommitGitRequest commit; protected final R request; - private final List> futures = new ArrayList<>(); + private final List> futures; + + public EntitiesExportCtx(SecurityUser user, CommitGitRequest commit, R request) { + this.user = user; + this.commit = commit; + this.request = request; + this.futures = new ArrayList<>(); + } + + public EntitiesExportCtx(EntitiesExportCtx other) { + this.user = other.getUser(); + this.commit = other.getCommit(); + this.request = other.getRequest(); + this.futures = other.getFutures(); + } public void add(ListenableFuture future) { futures.add(future); @@ -34,4 +63,6 @@ public class EntitiesExportCtx { .exportCredentials(config.isSaveCredentials()) .build(); } + + public abstract EntityExportSettings getSettings(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java new file mode 100644 index 0000000000..80d90590a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.apache.commons.lang3.ObjectUtils; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +public class EntityTypeExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityType entityType; + @Getter + private final boolean overwrite; + @Getter + private final EntityExportSettings settings; + + public EntityTypeExportCtx(EntitiesExportCtx parent, EntityTypeVersionCreateConfig config, SyncStrategy defaultSyncStrategy, EntityType entityType) { + super(parent); + this.entityType = entityType; + this.settings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java index 600d70106b..d713a2f71b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; @@ -14,4 +29,5 @@ public class SimpleEntitiesExportCtx extends EntitiesExportCtx Date: Wed, 15 Jun 2022 10:45:04 +0300 Subject: [PATCH 230/262] Refactoring --- .../sync/ie/DefaultEntitiesExportImportService.java | 4 ++-- .../service/sync/ie/EntitiesExportImportService.java | 2 +- .../service/sync/ie/exporting/EntityExportService.java | 2 +- .../sync/ie/exporting/impl/BaseEntityExportService.java | 8 ++++---- .../ie/exporting/impl/DefaultEntityExportService.java | 7 ++++--- .../sync/ie/exporting/impl/DeviceExportService.java | 7 ++++--- .../sync/ie/exporting/impl/RuleChainExportService.java | 5 +++-- .../ie/exporting/impl/WidgetsBundleExportService.java | 5 +++-- .../sync/vc/DefaultEntitiesVersionControlService.java | 6 +++--- .../server/service/sync/vc/data/EntitiesExportCtx.java | 2 +- .../service/sync/vc/data/SimpleEntitiesExportCtx.java | 7 +++++-- 11 files changed, 31 insertions(+), 24 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 2fdf5bc6c7..807831ecff 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -72,7 +72,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } @@ -80,7 +80,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(ctx, entityId, exportSettings); + return exportService.getExportData(ctx, entityId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 44f0d84a3a..349509b09e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -31,7 +31,7 @@ import java.util.Comparator; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 4174951e63..ac08425c10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -25,6 +25,6 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; public interface EntityExportService, D extends EntityExportData> { - D getExportData(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index f6f72d6ddd..ce6425435b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -30,12 +30,12 @@ import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { - setRelatedEntities(ctx.getTenantId(), entity, (D) exportData, exportSettings); - super.setAdditionalExportData(ctx, entity, exportData, exportSettings); + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + setRelatedEntities(ctx, entity, (D) exportData); + super.setAdditionalExportData(ctx, entity, exportData); } - protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData, EntityExportSettings settings) {} + protected void setRelatedEntities(EntitiesExportCtx ctx, E mainEntity, D exportData) {} protected abstract D newExportData(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 8dfdbb26eb..142fae385a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -60,7 +60,7 @@ public class DefaultEntityExportService ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public final D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { D exportData = newExportData(); E entity = exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entityId); @@ -70,12 +70,13 @@ public class DefaultEntityExportService ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + var exportSettings = ctx.getSettings(); if (exportSettings.isExportRelations()) { List relations = exportRelations(ctx, entity); exportData.setRelations(relations); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 0aa8933a27..33650912ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; @@ -36,9 +37,9 @@ public class DeviceExportService extends BaseEntityExportService ctx, Device device, DeviceExportData exportData) { + if (ctx.getSettings().isExportCredentials()) { + exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index a98f9954c8..9af29cceba 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; @@ -36,8 +37,8 @@ public class RuleChainExportService extends BaseEntityExportService ctx, RuleChain ruleChain, RuleChainExportData exportData) { + exportData.setMetaData(ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java index bb7f5c08a1..3065aae451 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.List; import java.util.Set; @@ -38,12 +39,12 @@ public class WidgetsBundleExportService extends BaseEntityExportService ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData) { if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { throw new IllegalArgumentException("Export of system Widget Bundles is not allowed"); } - List widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(tenantId, widgetsBundle.getAlias()); + List widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(ctx.getTenantId(), widgetsBundle.getAlias()); exportData.setWidgets(widgets); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6dd7c87dae..6a6d2e2e6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -182,7 +182,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId) throws Exception { - EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId, ctx.getSettings()); + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId); return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); } @@ -408,12 +408,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> { - SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null); - EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId, EntityExportSettings.builder() + SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder() .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) .build()); + EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId); return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), JacksonUtil.toPrettyString(currentVersion.sort()), JacksonUtil.toPrettyString(otherVersion.sort())), diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java index 041567dd07..130a09f255 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -41,7 +41,7 @@ public abstract class EntitiesExportCtx { this.futures = new ArrayList<>(); } - public EntitiesExportCtx(EntitiesExportCtx other) { + protected EntitiesExportCtx(EntitiesExportCtx other) { this.user = other.getUser(); this.commit = other.getCommit(); this.request = other.getRequest(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java index d713a2f71b..8cb9cc555e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -26,8 +26,11 @@ public class SimpleEntitiesExportCtx extends EntitiesExportCtx Date: Wed, 15 Jun 2022 11:09:08 +0300 Subject: [PATCH 231/262] Performance improvement for entities export --- .../ie/exporting/impl/AssetExportService.java | 2 +- .../impl/DashboardExportService.java | 2 +- .../impl/DefaultEntityExportService.java | 18 +++++++++++------ .../exporting/impl/DeviceExportService.java | 4 ++-- .../impl/DeviceProfileExportService.java | 4 ++-- .../impl/EntityViewExportService.java | 4 ++-- .../sync/vc/data/EntitiesExportCtx.java | 20 +++++++++++++++++++ 7 files changed, 40 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index e4daec3e3c..3c60fd2976 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -34,7 +34,7 @@ public class AssetExportService extends BaseEntityExportService ctx, Asset asset, EntityExportData exportData) { - asset.setCustomerId(getExternalIdOrElseInternal(asset.getCustomerId())); + asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 8b2c669bd2..1064962082 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -36,7 +36,7 @@ public class DashboardExportService extends BaseEntityExportService ctx, Dashboard dashboard, EntityExportData exportData) { if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) { dashboard.getAssignedCustomers().forEach(customerInfo -> { - customerInfo.setCustomerId(getExternalIdOrElseInternal(customerInfo.getCustomerId())); + customerInfo.setCustomerId(getExternalIdOrElseInternal(ctx, customerInfo.getCustomerId())); }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 50dee59fa7..1a1aec32c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -53,7 +53,8 @@ import java.util.stream.Collectors; @Primary public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { - @Autowired @Lazy + @Autowired + @Lazy protected ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; @@ -82,8 +83,8 @@ public class DefaultEntityExportService relations = exportRelations(ctx, entity); relations.forEach(relation -> { - relation.setFrom(getExternalIdOrElseInternal(relation.getFrom())); - relation.setTo(getExternalIdOrElseInternal(relation.getTo())); + relation.setFrom(getExternalIdOrElseInternal(ctx, relation.getFrom())); + relation.setTo(getExternalIdOrElseInternal(ctx, relation.getTo())); }); exportData.setRelations(relations); } @@ -134,10 +135,15 @@ public class DefaultEntityExportService ID getExternalIdOrElseInternal(ID internalId) { + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { if (internalId == null || internalId.isNullUid()) return internalId; - return Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) - .orElse(internalId); + var result = ctx.getExternalId(internalId); + if (result == null) { + result = Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) + .orElse(internalId); + ctx.putExternalId(internalId, result); + } + return result; } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 4ca7a3ff18..f3a8141e0d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -38,8 +38,8 @@ public class DeviceExportService extends BaseEntityExportService ctx, Device device, DeviceExportData exportData) { - device.setCustomerId(getExternalIdOrElseInternal(device.getCustomerId())); - device.setDeviceProfileId(getExternalIdOrElseInternal(device.getDeviceProfileId())); + device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId())); + device.setDeviceProfileId(getExternalIdOrElseInternal(ctx, device.getDeviceProfileId())); if (ctx.getSettings().isExportCredentials()) { exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId())); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java index c6e0d2d363..3e076ddc13 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java @@ -33,8 +33,8 @@ public class DeviceProfileExportService extends BaseEntityExportService ctx, DeviceProfile deviceProfile, EntityExportData exportData) { - deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(deviceProfile.getDefaultDashboardId())); - deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); + deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java index aebb284157..1f3459f18f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java @@ -33,8 +33,8 @@ public class EntityViewExportService extends BaseEntityExportService ctx, EntityView entityView, EntityExportData exportData) { - entityView.setEntityId(getExternalIdOrElseInternal(entityView.getEntityId())); - entityView.setCustomerId(getExternalIdOrElseInternal(entityView.getCustomerId())); + entityView.setEntityId(getExternalIdOrElseInternal(ctx, entityView.getEntityId())); + entityView.setCustomerId(getExternalIdOrElseInternal(ctx, entityView.getCustomerId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java index 130a09f255..c57c9f6d79 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; @@ -24,8 +26,11 @@ import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRe import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +@Slf4j @Data public abstract class EntitiesExportCtx { @@ -33,12 +38,14 @@ public abstract class EntitiesExportCtx { protected final CommitGitRequest commit; protected final R request; private final List> futures; + private final Map externalIdMap; public EntitiesExportCtx(SecurityUser user, CommitGitRequest commit, R request) { this.user = user; this.commit = commit; this.request = request; this.futures = new ArrayList<>(); + this.externalIdMap = new HashMap<>(); } protected EntitiesExportCtx(EntitiesExportCtx other) { @@ -46,6 +53,7 @@ public abstract class EntitiesExportCtx { this.commit = other.getCommit(); this.request = other.getRequest(); this.futures = other.getFutures(); + this.externalIdMap = other.getExternalIdMap(); } public void add(ListenableFuture future) { @@ -65,4 +73,16 @@ public abstract class EntitiesExportCtx { } public abstract EntityExportSettings getSettings(); + + @SuppressWarnings("unchecked") + public ID getExternalId(ID internalId) { + var result = externalIdMap.get(internalId); + log.debug("[{}][{}] Local cache {} for id", internalId.getEntityType(), internalId.getId(), result != null ? "hit" : "miss"); + return (ID) result; + } + + public void putExternalId(EntityId internalId, EntityId externalId) { + log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); + externalIdMap.put(internalId, externalId); + } } From 6579807bb68cee2de3aa0c6037346ad8ab210270 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 15 Jun 2022 11:44:23 +0300 Subject: [PATCH 232/262] Bug fixes and no more tenant id in the git repo --- .../sync/ie/exporting/impl/DefaultEntityExportService.java | 4 +++- .../service/sync/ie/exporting/impl/DeviceExportService.java | 5 ++++- .../sync/vc/DefaultEntitiesVersionControlService.java | 3 +++ .../org/thingsboard/server/common/data/ExportableEntity.java | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 1a1aec32c0..4d5d7952d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -70,11 +70,13 @@ public class DefaultEntityExportService extends HasId, HasName { @@ -28,4 +29,6 @@ public interface ExportableEntity extends HasId, HasName long getCreatedTime(); void setCreatedTime(long createdTime); + void setTenantId(TenantId tenantId); + } From fe4ec688b0960e9ad2793587dc2d70745988aa22 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 15 Jun 2022 12:02:28 +0300 Subject: [PATCH 233/262] Put external Id into Context --- .../sync/ie/exporting/impl/DefaultEntityExportService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 4d5d7952d8..4f748ba9d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -74,7 +74,9 @@ public class DefaultEntityExportService Date: Wed, 15 Jun 2022 12:07:08 +0300 Subject: [PATCH 234/262] UI: Updated version node in pkg script --- msa/js-executor/package.json | 2 +- msa/web-ui/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index d1df447072..686e1cad7d 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -6,7 +6,7 @@ "main": "server.js", "bin": "server.js", "scripts": { - "pkg": "pkg -t node12-linux-x64,node12-win-x64 --out-path ./target . && node install.js", + "pkg": "pkg -t node16-linux-x64,node16-win-x64 --out-path ./target . && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon server.js", "start-prod": "NODE_ENV=production nodemon server.js" diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 81e0663613..4824bcc165 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -6,7 +6,7 @@ "main": "server.js", "bin": "server.js", "scripts": { - "pkg": "pkg -t node12-linux-x64,node12-win-x64 --out-path ./target . && node install.js", + "pkg": "pkg -t node16-linux-x64,node16-win-x64 --out-path ./target . && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "WEB_FOLDER=./target/web nodemon server.js", "start-prod": "NODE_ENV=production nodemon server.js" From 835e8d5d1eb1d8e430655858f8e255421e482f18 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 15 Jun 2022 12:57:14 +0300 Subject: [PATCH 235/262] refactoring: commit4 --- .../server/controller/AlarmController.java | 2 +- .../DefaultTbNotificationEntityService.java | 24 +++++++++++++++++-- .../entitiy/alarm/DefaultTbAlarmService.java | 10 ++++---- .../service/entitiy/alarm/TbAlarmService.java | 3 +-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 2651277ccc..939e9da8a0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -147,7 +147,7 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - return tbAlarmService.delete(alarm, getCurrentUser().getCustomerId(), getCurrentUser()); + return tbAlarmService.delete(alarm, getCurrentUser()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 707747ae91..c7e4f2cd0b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -52,7 +52,6 @@ import org.thingsboard.server.service.gateway_device.GatewayNotificationsService import org.thingsboard.server.service.security.model.SecurityUser; import java.util.List; -import java.util.Locale; @Slf4j @Service @@ -337,6 +336,27 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS } public static EdgeEventActionType edgeTypeByActionType(ActionType actionType) { - return EdgeEventActionType.valueOf(actionType.toString().toUpperCase(Locale.ENGLISH)); + switch (actionType) { + case ADDED: + return EdgeEventActionType.ADDED; + case UPDATED: + return EdgeEventActionType.UPDATED; + case ALARM_ACK: + return EdgeEventActionType.ALARM_ACK; + case ALARM_CLEAR: + return EdgeEventActionType.ALARM_CLEAR; + case DELETED: + return EdgeEventActionType.DELETED; + case RELATION_ADD_OR_UPDATE: + return EdgeEventActionType.RELATION_ADD_OR_UPDATE; + case RELATION_DELETED: + return EdgeEventActionType.RELATION_DELETED; + case ASSIGNED_TO_EDGE: + return EdgeEventActionType.ASSIGNED_TO_EDGE; + case UNASSIGNED_FROM_EDGE: + return EdgeEventActionType.UNASSIGNED_FROM_EDGE; + default: + return null; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 305fda2b2b..4930f54a5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -78,13 +77,12 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public Boolean delete(Alarm alarm, CustomerId customerId, SecurityUser user) throws ThingsboardException { - TenantId tenantId = alarm.getTenantId(); + public Boolean delete(Alarm alarm, SecurityUser user) throws ThingsboardException { try { - List relatedEdgeIds = findRelatedEdgeIds(tenantId, alarm.getOriginator()); - notificationEntityService.notifyDeleteAlarm(tenantId, alarm, alarm.getOriginator(), customerId, + List relatedEdgeIds = findRelatedEdgeIds(user.getTenantId(), alarm.getOriginator()); + notificationEntityService.notifyDeleteAlarm(user.getTenantId(), alarm, alarm.getOriginator(), user.getCustomerId(), relatedEdgeIds, user, JacksonUtil.OBJECT_MAPPER.writeValueAsString(alarm)); - return alarmService.deleteAlarm(tenantId, alarm.getId()).isSuccessful(); + return alarmService.deleteAlarm(user.getTenantId(), alarm.getId()).isSuccessful(); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index f17fcb3b70..7a6d7f6a81 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.entitiy.alarm; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbAlarmService { @@ -28,5 +27,5 @@ public interface TbAlarmService { void clear(Alarm alarm, SecurityUser user) throws ThingsboardException; - Boolean delete(Alarm alarm, CustomerId customerId, SecurityUser user) throws ThingsboardException; + Boolean delete(Alarm alarm, SecurityUser user) throws ThingsboardException; } From c4bdee92dbc0d04054a71f95f754ccb5278e7e69 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 15 Jun 2022 14:04:13 +0300 Subject: [PATCH 236/262] PSQL annotation and dialect cleanup. --- ...va => SqlEntityDatabaseSchemaService.java} | 6 +- ...e.java => SqlTsDatabaseSchemaService.java} | 6 +- ....java => SqlTsDatabaseUpgradeService.java} | 4 +- .../TimescaleTsDatabaseSchemaService.java | 6 - .../TimescaleTsDatabaseUpgradeService.java | 2 - .../src/main/resources/thingsboard.yml | 3 - ...> SqlEntityDatabaseSchemaServiceTest.java} | 16 +-- .../thingsboard/server/dao/util/HsqlDao.java | 26 ---- .../thingsboard/server/dao/util/PsqlDao.java | 26 ---- .../server/dao/util/PsqlTsLatestAnyDao.java | 27 ---- .../server/dao/HsqlTsLatestDaoConfig.java | 37 ----- .../server/dao/PsqlTsDaoConfig.java | 37 ----- ...qlTsDaoConfig.java => SqlTsDaoConfig.java} | 8 +- ...oConfig.java => SqlTsLatestDaoConfig.java} | 8 +- .../server/dao/TimescaleDaoConfig.java | 2 - .../dao/TimescaleTsLatestDaoConfig.java | 4 +- .../HsqlAttributesInsertRepository.java | 76 ---------- ...ava => SqlAttributesInsertRepository.java} | 4 +- ...qlComponentDescriptorInsertRepository.java | 65 --------- ...lComponentDescriptorInsertRepository.java} | 4 +- .../sql/edge/EdgeEventInsertRepository.java | 3 +- .../dao/sql/event/EventInsertRepository.java | 2 - ...ry.java => SqlEventCleanupRepository.java} | 4 +- .../HsqlRelationInsertRepository.java | 63 --------- ....java => SqlRelationInsertRepository.java} | 4 +- .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 62 -------- .../insert/hsql/HsqlInsertTsRepository.java | 87 ------------ .../hsql/HsqlLatestInsertTsRepository.java | 85 ----------- .../SqlLatestInsertTsRepository.java} | 8 +- .../SqlInsertTsRepository.java} | 6 +- .../SqlPartitioningRepository.java} | 10 +- .../TimescaleInsertTsRepository.java | 2 - .../JpaSqlTimeseriesDao.java} | 32 ++--- .../{PsqlPartition.java => SqlPartition.java} | 4 +- .../server/dao/AbstractDaoServiceTest.java | 2 +- .../server/dao/AbstractJpaDaoTest.java | 2 +- .../server/dao/util/DaoTestUtil.java | 40 ------ .../server/dao/util/SqlDbType.java | 20 --- dao/src/test/resources/nosql-test.properties | 1 - dao/src/test/resources/sql-test.properties | 2 - docker/tb-node.hybrid.env | 1 - docker/tb-node.postgres.env | 1 - msa/tb/docker-cassandra/Dockerfile | 1 - msa/tb/docker-postgres/Dockerfile | 1 - msa/tb/docker-tb/Dockerfile | 61 -------- msa/tb/docker-tb/start-db.sh | 18 --- msa/tb/docker-tb/stop-db.sh | 18 --- msa/tb/pom.xml | 132 +----------------- pom.xml | 6 - 49 files changed, 55 insertions(+), 990 deletions(-) rename application/src/main/java/org/thingsboard/server/service/install/{PsqlEntityDatabaseSchemaService.java => SqlEntityDatabaseSchemaService.java} (88%) rename application/src/main/java/org/thingsboard/server/service/install/{PsqlTsDatabaseSchemaService.java => SqlTsDatabaseSchemaService.java} (85%) rename application/src/main/java/org/thingsboard/server/service/install/{PsqlTsDatabaseUpgradeService.java => SqlTsDatabaseUpgradeService.java} (98%) rename application/src/test/java/org/thingsboard/server/service/install/{PsqlEntityDatabaseSchemaServiceTest.java => SqlEntityDatabaseSchemaServiceTest.java} (66%) delete mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java delete mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java delete mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java rename dao/src/main/java/org/thingsboard/server/dao/{HsqlTsDaoConfig.java => SqlTsDaoConfig.java} (87%) rename dao/src/main/java/org/thingsboard/server/dao/{PsqlTsLatestDaoConfig.java => SqlTsLatestDaoConfig.java} (86%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sql/attributes/{PsqlAttributesInsertRepository.java => SqlAttributesInsertRepository.java} (85%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sql/component/{PsqlComponentDescriptorInsertRepository.java => SqlComponentDescriptorInsertRepository.java} (94%) rename dao/src/main/java/org/thingsboard/server/dao/sql/event/{PsqlEventCleanupRepository.java => SqlEventCleanupRepository.java} (91%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sql/relation/{PsqlRelationInsertRepository.java => SqlRelationInsertRepository.java} (90%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/{psql/PsqlLatestInsertTsRepository.java => sql/SqlLatestInsertTsRepository.java} (96%) rename dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/{psql/PsqlInsertTsRepository.java => sql/SqlInsertTsRepository.java} (93%) rename dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/{psql/PsqlPartitioningRepository.java => sql/SqlPartitioningRepository.java} (80%) rename dao/src/main/java/org/thingsboard/server/dao/sqlts/{psql/JpaPsqlTimeseriesDao.java => sql/JpaSqlTimeseriesDao.java} (85%) rename dao/src/main/java/org/thingsboard/server/dao/timeseries/{PsqlPartition.java => SqlPartition.java} (92%) delete mode 100644 dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java delete mode 100644 dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java delete mode 100644 msa/tb/docker-tb/Dockerfile delete mode 100644 msa/tb/docker-tb/start-db.sh delete mode 100644 msa/tb/docker-tb/stop-db.sh diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java index eaa9d708be..7ae155402a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java @@ -18,19 +18,17 @@ package org.thingsboard.server.service.install; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; @Service -@PsqlDao @Profile("install") @Slf4j -public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService +public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService { public static final String SCHEMA_ENTITIES_SQL = "schema-entities.sql"; public static final String SCHEMA_ENTITIES_IDX_SQL = "schema-entities-idx.sql"; public static final String SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL = "schema-entities-idx-psql-addon.sql"; - public PsqlEntityDatabaseSchemaService() { + public SqlEntityDatabaseSchemaService() { super(SCHEMA_ENTITIES_SQL, SCHEMA_ENTITIES_IDX_SQL); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java index 0e7cff7d63..f3ede624ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java @@ -18,19 +18,17 @@ package org.thingsboard.server.service.install; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @Service @SqlTsDao -@PsqlDao @Profile("install") -public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { +public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") private String partitionType; - public PsqlTsDatabaseSchemaService() { + public SqlTsDatabaseSchemaService() { super("schema-ts-psql.sql", null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java rename to application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java index 55fa73fb27..c1d41a6f6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; import java.io.File; @@ -36,8 +35,7 @@ import java.sql.DriverManager; @Profile("install") @Slf4j @SqlTsDao -@PsqlDao -public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { +public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") private String partitionType; diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java index a0257ba0e4..409498cfd3 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java @@ -19,16 +19,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - @Service @TimescaleDBTsDao -@PsqlDao @Profile("install") @Slf4j public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 1834ca6552..245dc4503d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -22,7 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import java.io.File; @@ -37,7 +36,6 @@ import java.sql.DriverManager; @Profile("install") @Slf4j @TimescaleDBTsDao -@PsqlDao public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { @Value("${sql.timescale.chunk_time_interval:86400000}") diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4f11b6880c..9f6f3099ef 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -176,8 +176,6 @@ database: ts_latest: type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) -# note: timescale works only with postgreSQL database for DATABASE_ENTITIES_TYPE. - # Cassandra driver configuration parameters cassandra: # Thingsboard cluster name @@ -535,7 +533,6 @@ spring: open-in-view: "false" hibernate: ddl-auto: "none" - database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQL10Dialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" diff --git a/application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java b/application/src/test/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaServiceTest.java similarity index 66% rename from application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaServiceTest.java index eaea437bc7..07a32d5cdc 100644 --- a/application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaServiceTest.java @@ -23,31 +23,31 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -public class PsqlEntityDatabaseSchemaServiceTest { +public class SqlEntityDatabaseSchemaServiceTest { @Test public void givenPsqlDbSchemaService_whenCreateDatabaseSchema_thenVerifyPsqlIndexSpecificCall() throws Exception { - PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService()); + SqlEntityDatabaseSchemaService service = spy(new SqlEntityDatabaseSchemaService()); willDoNothing().given(service).executeQueryFromFile(anyString()); service.createDatabaseSchema(); verify(service, times(1)).createDatabaseIndexes(); - verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_SQL); - verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); - verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + verify(service, times(1)).executeQueryFromFile(SqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_SQL); + verify(service, times(1)).executeQueryFromFile(SqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); + verify(service, times(1)).executeQueryFromFile(SqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); verify(service, times(3)).executeQueryFromFile(anyString()); } @Test public void givenPsqlDbSchemaService_whenCreateDatabaseIndexes_thenVerifyPsqlIndexSpecificCall() throws Exception { - PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService()); + SqlEntityDatabaseSchemaService service = spy(new SqlEntityDatabaseSchemaService()); willDoNothing().given(service).executeQueryFromFile(anyString()); service.createDatabaseIndexes(); - verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); - verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + verify(service, times(1)).executeQueryFromFile(SqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); + verify(service, times(1)).executeQueryFromFile(SqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); verify(service, times(2)).executeQueryFromFile(anyString()); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java deleted file mode 100644 index 949c8d3569..0000000000 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") -public @interface HsqlDao { -} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java deleted file mode 100644 index ba30f8975a..0000000000 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQL10Dialect") -public @interface PsqlDao { -} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java deleted file mode 100644 index 368f550c14..0000000000 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') " + - "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQL10Dialect'") -public @interface PsqlTsLatestAnyDao { -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java deleted file mode 100644 index 2012ca5a82..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsLatestDao; -import org.thingsboard.server.dao.util.TbAutoConfiguration; - -@Configuration -@TbAutoConfiguration -@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.hsql", "org.thingsboard.server.dao.sqlts.latest"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) -@EnableTransactionManagement -@SqlTsLatestDao -@HsqlDao -public class HsqlTsLatestDaoConfig { - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java deleted file mode 100644 index 06c8cf4a5b..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; -import org.thingsboard.server.dao.util.TbAutoConfiguration; - -@Configuration -@TbAutoConfiguration -@ComponentScan({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.insert.psql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.psql"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) -@EnableTransactionManagement -@PsqlDao -@SqlTsDao -public class PsqlTsDaoConfig { - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java similarity index 87% rename from dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java rename to dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java index 932312f96e..cd1401d60c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java @@ -20,18 +20,16 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; import org.thingsboard.server.dao.util.TbAutoConfiguration; @Configuration @TbAutoConfiguration -@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.hsql"}) +@ComponentScan({"org.thingsboard.server.dao.sqlts.sql", "org.thingsboard.server.dao.sqlts.insert.sql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.sql"}) @EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) @EnableTransactionManagement @SqlTsDao -@HsqlDao -public class HsqlTsDaoConfig { +public class SqlTsDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTsLatestDaoConfig.java similarity index 86% rename from dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java rename to dao/src/main/java/org/thingsboard/server/dao/SqlTsLatestDaoConfig.java index 9d8d625e78..4fe4aaed43 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTsLatestDaoConfig.java @@ -20,18 +20,16 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsLatestDao; import org.thingsboard.server.dao.util.TbAutoConfiguration; @Configuration @TbAutoConfiguration -@ComponentScan({"org.thingsboard.server.dao.sqlts.psql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@ComponentScan({"org.thingsboard.server.dao.sqlts.sql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.sql", "org.thingsboard.server.dao.sqlts.latest"}) @EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) @EnableTransactionManagement @SqlTsLatestDao -@PsqlDao -public class PsqlTsLatestDaoConfig { +public class SqlTsLatestDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java index 0e4d4750f9..6b11ff8005 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java @@ -20,7 +20,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TbAutoConfiguration; import org.thingsboard.server.dao.util.TimescaleDBTsDao; @@ -31,7 +30,6 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; @EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale"}) @EnableTransactionManagement @TimescaleDBTsDao -@PsqlDao public class TimescaleDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java index 02082085ba..3fd452a1b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java @@ -20,18 +20,16 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TbAutoConfiguration; import org.thingsboard.server.dao.util.TimescaleDBTsLatestDao; @Configuration @TbAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.timescale"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.sql", "org.thingsboard.server.dao.sqlts.latest"}) @EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) @EnableTransactionManagement @TimescaleDBTsLatestDao -@PsqlDao public class TimescaleTsLatestDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java deleted file mode 100644 index 24465e0742..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.attributes; - -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.HsqlDao; - -import java.sql.Types; -import java.util.List; - -@HsqlDao -@Repository -@Transactional -public class HsqlAttributesInsertRepository extends AttributeKvInsertRepository { - - private static final String INSERT_OR_UPDATE = - "MERGE INTO attribute_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + - "A (entity_type, entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + - "ON (attribute_kv.entity_type=A.entity_type " + - "AND attribute_kv.entity_id=A.entity_id " + - "AND attribute_kv.attribute_type=A.attribute_type " + - "AND attribute_kv.attribute_key=A.attribute_key) " + - "WHEN MATCHED THEN UPDATE SET attribute_kv.str_v = A.str_v, attribute_kv.long_v = A.long_v, attribute_kv.dbl_v = A.dbl_v, attribute_kv.bool_v = A.bool_v, attribute_kv.json_v = A.json_v, attribute_kv.last_update_ts = A.last_update_ts " + - "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + - "VALUES (A.entity_type, A.entity_id, A.attribute_type, A.attribute_key, A.str_v, A.long_v, A.dbl_v, A.bool_v, A.json_v, A.last_update_ts)"; - - @Override - protected void saveOrUpdate(List entities) { - entities.forEach(entity -> { - jdbcTemplate.update(INSERT_OR_UPDATE, ps -> { - ps.setString(1, entity.getId().getEntityType().name()); - ps.setObject(2, entity.getId().getEntityId()); - ps.setString(3, entity.getId().getAttributeType()); - ps.setString(4, entity.getId().getAttributeKey()); - ps.setString(5, entity.getStrValue()); - - if (entity.getLongValue() != null) { - ps.setLong(6, entity.getLongValue()); - } else { - ps.setNull(6, Types.BIGINT); - } - - if (entity.getDoubleValue() != null) { - ps.setDouble(7, entity.getDoubleValue()); - } else { - ps.setNull(7, Types.DOUBLE); - } - - if (entity.getBooleanValue() != null) { - ps.setBoolean(8, entity.getBooleanValue()); - } else { - ps.setNull(8, Types.BOOLEAN); - } - - ps.setString(9, entity.getJsonValue()); - - ps.setLong(10, entity.getLastUpdateTs()); - }); - }); - } -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java similarity index 85% rename from dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java index 2b5b53691a..e18f5178a8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java @@ -17,11 +17,9 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.util.PsqlDao; -@PsqlDao @Repository @Transactional -public class PsqlAttributesInsertRepository extends AttributeKvInsertRepository { +public class SqlAttributesInsertRepository extends AttributeKvInsertRepository { } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java deleted file mode 100644 index e735596de3..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.component; - -import org.springframework.stereotype.Repository; -import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; -import org.thingsboard.server.dao.util.HsqlDao; - -import javax.persistence.Query; - -@HsqlDao -@Repository -public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { - - private static final String P_KEY_CONFLICT_STATEMENT = "(component_descriptor.id=UUID(I.id))"; - private static final String UNQ_KEY_CONFLICT_STATEMENT = "(component_descriptor.clazz=I.clazz)"; - - private static final String INSERT_OR_UPDATE_ON_P_KEY_CONFLICT = getInsertString(P_KEY_CONFLICT_STATEMENT); - private static final String INSERT_OR_UPDATE_ON_UNQ_KEY_CONFLICT = getInsertString(UNQ_KEY_CONFLICT_STATEMENT); - - @Override - public ComponentDescriptorEntity saveOrUpdate(ComponentDescriptorEntity entity) { - return saveAndGet(entity, INSERT_OR_UPDATE_ON_P_KEY_CONFLICT, INSERT_OR_UPDATE_ON_UNQ_KEY_CONFLICT); - } - - @Override - protected Query getQuery(ComponentDescriptorEntity entity, String query) { - return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", entity.getUuid().toString()) - .setParameter("created_time", entity.getCreatedTime()) - .setParameter("actions", entity.getActions()) - .setParameter("clazz", entity.getClazz()) - .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) - .setParameter("name", entity.getName()) - .setParameter("scope", entity.getScope().name()) - .setParameter("search_text", entity.getSearchText()) - .setParameter("type", entity.getType().name()); - } - - @Override - protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { - getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, entity.getUuid()); - } - - private static String getInsertString(String conflictStatement) { - return "MERGE INTO component_descriptor USING (VALUES :id, :created_time, :actions, :clazz, :configuration_descriptor, :name, :scope, :search_text, :type) I (id, created_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) ON " - + conflictStatement - + " WHEN MATCHED THEN UPDATE SET component_descriptor.id = UUID(I.id), component_descriptor.actions = I.actions, component_descriptor.clazz = I.clazz, component_descriptor.configuration_descriptor = I.configuration_descriptor, component_descriptor.name = I.name, component_descriptor.scope = I.scope, component_descriptor.search_text = I.search_text, component_descriptor.type = I.type" + - " WHEN NOT MATCHED THEN INSERT (id, created_time, actions, clazz, configuration_descriptor, name, scope, search_text, type) VALUES (UUID(I.id), I.created_time, I.actions, I.clazz, I.configuration_descriptor, I.name, I.scope, I.search_text, I.type)"; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/SqlComponentDescriptorInsertRepository.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/component/SqlComponentDescriptorInsertRepository.java index 1886dd3b35..593cccee2a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/SqlComponentDescriptorInsertRepository.java @@ -17,11 +17,9 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; -import org.thingsboard.server.dao.util.PsqlDao; -@PsqlDao @Repository -public class PsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { +public class SqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { private static final String ID = "id = :id"; private static final String CLAZZ_CLAZZ = "clazz = :clazz"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java index c67aaabbb1..694d68556f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java @@ -24,13 +24,12 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.EdgeEventEntity; -import org.thingsboard.server.dao.util.PsqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; -@PsqlDao + @Repository @Transactional public class EdgeEventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 1f2d4c01be..6234db58da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,14 +25,12 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.EventEntity; -import org.thingsboard.server.dao.util.PsqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; import java.util.regex.Pattern; -@PsqlDao @Repository @Transactional public class EventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventCleanupRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index 664a148948..6a17f7c740 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.event; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.util.PsqlDao; import java.sql.Connection; import java.sql.PreparedStatement; @@ -27,9 +26,8 @@ import java.sql.SQLException; import java.util.concurrent.TimeUnit; @Slf4j -@PsqlDao @Repository -public class PsqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository { +public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository { @Override public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java deleted file mode 100644 index 5da0a8a2de..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sql.RelationCompositeKey; -import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.HsqlDao; - -import javax.persistence.Query; - -@HsqlDao -@Repository -@Transactional -public class HsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { - - private static final String INSERT_ON_CONFLICT_DO_UPDATE = "MERGE INTO relation USING (VALUES :fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) R " + - "(from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " + - "ON (relation.from_id = UUID(R.from_id) AND relation.from_type = R.from_type AND relation.relation_type_group = R.relation_type_group AND relation.relation_type = R.relation_type AND relation.to_id = UUID(R.to_id) AND relation.to_type = R.to_type) " + - "WHEN MATCHED THEN UPDATE SET relation.additional_info = R.additional_info " + - "WHEN NOT MATCHED THEN INSERT (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) VALUES (UUID(R.from_id), R.from_type, UUID(R.to_id), R.to_type, R.relation_type_group, R.relation_type, R.additional_info)"; - - protected Query getQuery(RelationEntity entity, String query) { - Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); - if (entity.getAdditionalInfo() == null) { - nativeQuery.setParameter("additionalInfo", null); - } else { - nativeQuery.setParameter("additionalInfo", entity.getAdditionalInfo().toString()); - } - return nativeQuery - .setParameter("fromId", entity.getFromId().toString()) - .setParameter("fromType", entity.getFromType()) - .setParameter("toId", entity.getToId().toString()) - .setParameter("toType", entity.getToType()) - .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) - .setParameter("relationType", entity.getRelationType()); - } - - @Override - public RelationEntity saveOrUpdate(RelationEntity entity) { - return processSaveOrUpdate(entity); - } - - @Override - protected RelationEntity processSaveOrUpdate(RelationEntity entity) { - getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE).executeUpdate(); - return entityManager.find(RelationEntity.class, new RelationCompositeKey(entity.toData())); - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java similarity index 90% rename from dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index a622d19e6d..23d24588b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -18,12 +18,10 @@ package org.thingsboard.server.dao.sql.relation; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.PsqlDao; -@PsqlDao @Repository @Transactional -public class PsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { +public class SqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java deleted file mode 100644 index 770c9142d4..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.hsql; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sqlts.AbstractChunkedAggregationTimeseriesDao; -import org.thingsboard.server.dao.timeseries.TimeseriesDao; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; - - -@Component -@Slf4j -@SqlTsDao -@HsqlDao -public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao implements TimeseriesDao { - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); - String strKey = tsKvEntry.getKey(); - Integer keyId = getOrSaveKeyId(strKey); - TsKvEntity entity = new TsKvEntity(); - entity.setEntityId(entityId.getId()); - entity.setTs(tsKvEntry.getTs()); - entity.setKey(keyId); - entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - log.trace("Saving entity: {}", entity); - return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); - } - - @Override - public void cleanup(long systemTtl) { - - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java deleted file mode 100644 index 596c913ed8..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.insert.hsql; - -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; -import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; - -@SqlTsDao -@HsqlDao -@Repository -@Transactional -public class HsqlInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository { - - private static final String INSERT_OR_UPDATE = - "MERGE INTO ts_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + - "T (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + - "ON (ts_kv.entity_id=T.entity_id " + - "AND ts_kv.key=T.key " + - "AND ts_kv.ts=T.ts) " + - "WHEN MATCHED THEN UPDATE SET ts_kv.bool_v = T.bool_v, ts_kv.str_v = T.str_v, ts_kv.long_v = T.long_v, ts_kv.dbl_v = T.dbl_v ,ts_kv.json_v = T.json_v " + - "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + - "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v, T.json_v);"; - - @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvEntity tsKvEntity = entities.get(i); - ps.setObject(1, tsKvEntity.getEntityId()); - ps.setInt(2, tsKvEntity.getKey()); - ps.setLong(3, tsKvEntity.getTs()); - - if (tsKvEntity.getBooleanValue() != null) { - ps.setBoolean(4, tsKvEntity.getBooleanValue()); - } else { - ps.setNull(4, Types.BOOLEAN); - } - - ps.setString(5, tsKvEntity.getStrValue()); - - if (tsKvEntity.getLongValue() != null) { - ps.setLong(6, tsKvEntity.getLongValue()); - } else { - ps.setNull(6, Types.BIGINT); - } - - if (tsKvEntity.getDoubleValue() != null) { - ps.setDouble(7, tsKvEntity.getDoubleValue()); - } else { - ps.setNull(7, Types.DOUBLE); - } - - ps.setString(8, tsKvEntity.getJsonValue()); - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java deleted file mode 100644 index c69bb6b1d5..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.insert.latest.hsql; - -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; -import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsLatestDao; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; - -@SqlTsLatestDao -@HsqlDao -@Repository -@Transactional -public class HsqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { - - private static final String INSERT_OR_UPDATE = - "MERGE INTO ts_kv_latest USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + - "T (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + - "ON (ts_kv_latest.entity_id=T.entity_id " + - "AND ts_kv_latest.key=T.key) " + - "WHEN MATCHED THEN UPDATE SET ts_kv_latest.ts = T.ts, ts_kv_latest.bool_v = T.bool_v, ts_kv_latest.str_v = T.str_v, ts_kv_latest.long_v = T.long_v, ts_kv_latest.dbl_v = T.dbl_v, ts_kv_latest.json_v = T.json_v " + - "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + - "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v, T.json_v);"; - - @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - ps.setObject(1, entities.get(i).getEntityId()); - ps.setInt(2, entities.get(i).getKey()); - ps.setLong(3, entities.get(i).getTs()); - - if (entities.get(i).getBooleanValue() != null) { - ps.setBoolean(4, entities.get(i).getBooleanValue()); - } else { - ps.setNull(4, Types.BOOLEAN); - } - - ps.setString(5, entities.get(i).getStrValue()); - - if (entities.get(i).getLongValue() != null) { - ps.setLong(6, entities.get(i).getLongValue()); - } else { - ps.setNull(6, Types.BIGINT); - } - - if (entities.get(i).getDoubleValue() != null) { - ps.setDouble(7, entities.get(i).getDoubleValue()); - } else { - ps.setNull(7, Types.DOUBLE); - } - - ps.setString(8, entities.get(i).getJsonValue()); - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 726cf56826..451ba29987 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.insert.latest.psql; +package org.thingsboard.server.dao.sqlts.insert.latest.sql; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.BatchPreparedStatementSetter; @@ -24,7 +24,7 @@ import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.PsqlTsLatestAnyDao; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -33,10 +33,10 @@ import java.util.ArrayList; import java.util.List; -@PsqlTsLatestAnyDao +@SqlTsLatestAnyDao @Repository @Transactional -public class PsqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { +public class SqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") private Boolean updateByLatestTs; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlInsertTsRepository.java similarity index 93% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlInsertTsRepository.java index adae2c8f3b..c61c852d63 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlInsertTsRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.insert.psql; +package org.thingsboard.server.dao.sqlts.insert.sql; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.stereotype.Repository; @@ -21,7 +21,6 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; import java.sql.PreparedStatement; @@ -30,10 +29,9 @@ import java.sql.Types; import java.util.List; @SqlTsDao -@PsqlDao @Repository @Transactional -public class PsqlInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository { +public class SqlInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository { private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO ts_kv (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES (?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java similarity index 80% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlPartitioningRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java index 5821b0f7c8..899a195538 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlPartitioningRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java @@ -13,27 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.insert.psql; +package org.thingsboard.server.dao.sqlts.insert.sql; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.timeseries.PsqlPartition; -import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.timeseries.SqlPartition; import org.thingsboard.server.dao.util.SqlTsDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @SqlTsDao -@PsqlDao @Repository @Transactional -public class PsqlPartitioningRepository { +public class SqlPartitioningRepository { @PersistenceContext private EntityManager entityManager; - public void save(PsqlPartition partition) { + public void save(SqlPartition partition) { entityManager.createNativeQuery(partition.getQuery()) .executeUpdate(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java index ae8f5d8376..02e71c1c7d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java @@ -21,7 +21,6 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import java.sql.PreparedStatement; @@ -30,7 +29,6 @@ import java.sql.Types; import java.util.List; @TimescaleDBTsDao -@PsqlDao @Repository @Transactional public class TimescaleInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java similarity index 85% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java index 22639dab5a..ce173e046b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.psql; +package org.thingsboard.server.dao.sqlts.sql; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -29,10 +29,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.AbstractChunkedAggregationTimeseriesDao; -import org.thingsboard.server.dao.sqlts.insert.psql.PsqlPartitioningRepository; -import org.thingsboard.server.dao.timeseries.PsqlPartition; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.timeseries.SqlPartition; import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; -import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; import java.sql.Connection; @@ -52,15 +51,14 @@ import java.util.concurrent.locks.ReentrantLock; @Component @Slf4j -@PsqlDao @SqlTsDao -public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao { +public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao { - private final Map partitions = new ConcurrentHashMap<>(); + private final Map partitions = new ConcurrentHashMap<>(); private static final ReentrantLock partitionCreationLock = new ReentrantLock(); @Autowired - private PsqlPartitioningRepository partitioningRepository; + private SqlPartitioningRepository partitioningRepository; private SqlTsPartitionDate tsFormat; @@ -134,24 +132,24 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa long partitionEndTs = toMills(localDateTimeEnd); ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); - savePartition(new PsqlPartition(partitionStartTs, partitionEndTs, partitionDate)); + savePartition(new SqlPartition(partitionStartTs, partitionEndTs, partitionDate)); } } } - private void savePartition(PsqlPartition psqlPartition) { - if (!partitions.containsKey(psqlPartition.getStart())) { + private void savePartition(SqlPartition sqlPartition) { + if (!partitions.containsKey(sqlPartition.getStart())) { partitionCreationLock.lock(); try { - log.trace("Saving partition: {}", psqlPartition); - partitioningRepository.save(psqlPartition); - log.trace("Adding partition to Set: {}", psqlPartition); - partitions.put(psqlPartition.getStart(), psqlPartition); + log.trace("Saving partition: {}", sqlPartition); + partitioningRepository.save(sqlPartition); + log.trace("Adding partition to Set: {}", sqlPartition); + partitions.put(sqlPartition.getStart(), sqlPartition); } catch (DataIntegrityViolationException ex) { log.trace("Error occurred during partition save:", ex); if (ex.getCause() instanceof ConstraintViolationException) { - log.warn("Saving partition [{}] rejected. Timeseries data will save to the ts_kv_indefinite (DEFAULT) partition.", psqlPartition.getPartitionDate()); - partitions.put(psqlPartition.getStart(), psqlPartition); + log.warn("Saving partition [{}] rejected. Timeseries data will save to the ts_kv_indefinite (DEFAULT) partition.", sqlPartition.getPartitionDate()); + partitions.put(sqlPartition.getStart(), sqlPartition); } else { throw new RuntimeException(ex); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java similarity index 92% rename from dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java rename to dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java index fe3019ea25..5c47689b6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.timeseries; import lombok.Data; @Data -public class PsqlPartition { +public class SqlPartition { private static final String TABLE_REGEX = "ts_kv_"; @@ -27,7 +27,7 @@ public class PsqlPartition { private String partitionDate; private String query; - public PsqlPartition(long start, long end, String partitionDate) { + public SqlPartition(long start, long end, String partitionDate) { this.start = start; this.end = end; this.partitionDate = partitionDate; diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java index d54933a19d..9d5e080e2a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.service.DaoSqlTest; @RunWith(SpringRunner.class) -@ContextConfiguration(classes = {JpaDaoConfig.class, PsqlTsDaoConfig.class, PsqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class}) +@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class}) @DaoSqlTest @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @TestExecutionListeners({ diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java index 5f7fc7caa0..d99692cd5a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java @@ -30,7 +30,7 @@ import org.thingsboard.server.dao.service.DaoSqlTest; * Created by Valerii Sosliuk on 4/22/2017. */ @RunWith(SpringRunner.class) -@ContextConfiguration(classes = {JpaDaoConfig.class, PsqlTsDaoConfig.class, PsqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class}) +@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class}) @DaoSqlTest @TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, diff --git a/dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java b/dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java deleted file mode 100644 index 34cceb440b..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.util; - -import org.springframework.jdbc.core.JdbcTemplate; - -import java.sql.DriverManager; - -public class DaoTestUtil { - private static final String POSTGRES_DRIVER_CLASS = "org.postgresql.Driver"; - private static final String H2_DRIVER_CLASS = "org.hsqldb.jdbc.JDBCDriver"; - - - public static SqlDbType getSqlDbType(JdbcTemplate template){ - try { - String driverName = DriverManager.getDriver(template.getDataSource().getConnection().getMetaData().getURL()).getClass().getName(); - if (POSTGRES_DRIVER_CLASS.equals(driverName)) { - return SqlDbType.POSTGRES; - } else if (H2_DRIVER_CLASS.equals(driverName)) { - return SqlDbType.H2; - } - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java b/dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java deleted file mode 100644 index f655955807..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.util; - -public enum SqlDbType { - POSTGRES, H2; -} diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 8f661359a9..1c6d2442cf 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -11,7 +11,6 @@ spring.jpa.properties.hibernate.jdbc.log.warnings=false spring.jpa.show-sql=false spring.jpa.hibernate.ddl-auto=none -spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index d843b1dc09..b2ded716c4 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -12,7 +12,6 @@ spring.jpa.properties.hibernate.jdbc.log.warnings=false spring.jpa.show-sql=false spring.jpa.hibernate.ddl-auto=none -spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb @@ -32,7 +31,6 @@ service.type=monolith #spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true #spring.jpa.show-sql=false #spring.jpa.hibernate.ddl-auto=none -#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL10Dialect # #spring.datasource.username=postgres #spring.datasource.password=postgres diff --git a/docker/tb-node.hybrid.env b/docker/tb-node.hybrid.env index 5f72c6e1a1..e03239bd96 100644 --- a/docker/tb-node.hybrid.env +++ b/docker/tb-node.hybrid.env @@ -2,7 +2,6 @@ DATABASE_TS_TYPE=cassandra CASSANDRA_URL=cassandra:9042 -SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard SPRING_DATASOURCE_USERNAME=postgres diff --git a/docker/tb-node.postgres.env b/docker/tb-node.postgres.env index f4e11cd70a..633b8b6fe9 100644 --- a/docker/tb-node.postgres.env +++ b/docker/tb-node.postgres.env @@ -1,7 +1,6 @@ # ThingsBoard server configuration for PostgreSQL database DATABASE_TS_TYPE=sql -SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard SPRING_DATASOURCE_USERNAME=postgres diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index 144c359cc0..5b22efd636 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -54,7 +54,6 @@ ENV DATABASE_TS_TYPE=cassandra ENV PGDATA=/data/db ENV CASSANDRA_DATA=/data/cassandra -ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect ENV SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver ENV SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/thingsboard ENV SPRING_DATASOURCE_USERNAME=${pkg.user} diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index 6b6dbc0011..720f770007 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -50,7 +50,6 @@ ENV DATABASE_TS_TYPE=sql ENV PGDATA=/data/db ENV PATH=$PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQL10Dialect ENV SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver ENV SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/thingsboard ENV SPRING_DATASOURCE_USERNAME=${pkg.user} diff --git a/msa/tb/docker-tb/Dockerfile b/msa/tb/docker-tb/Dockerfile deleted file mode 100644 index 6deea6d274..0000000000 --- a/msa/tb/docker-tb/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright © 2016-2022 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM thingsboard/openjdk11 - -COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-tb.sh /usr/bin \ - && mv /tmp/upgrade-tb.sh /usr/bin \ - && mv /tmp/install-tb.sh /usr/bin \ - && mv /tmp/start-db.sh /usr/bin \ - && mv /tmp/stop-db.sh /usr/bin - -RUN dpkg -i /tmp/${pkg.name}.deb -RUN rm /tmp/${pkg.name}.deb - -RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : - -RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \ - && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf - -ENV DATA_FOLDER=/data - -ENV HTTP_BIND_PORT=9090 -ENV DATABASE_TS_TYPE=sql - -ENV SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.HSQLDialect -ENV SPRING_DRIVER_CLASS_NAME=org.hsqldb.jdbc.JDBCDriver -ENV SPRING_DATASOURCE_URL=jdbc:hsqldb:file:/data/db/thingsboardDb;sql.enforce_size=false;hsqldb.log_size=5 -ENV SPRING_DATASOURCE_USERNAME=sa -ENV SPRING_DATASOURCE_PASSWORD= - -RUN mkdir -p /data -RUN chown -R ${pkg.user}:${pkg.user} /data - -RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar - -USER ${pkg.user} - -EXPOSE 9090 -EXPOSE 1883 -EXPOSE 5683/udp -EXPOSE 5685/udp - -VOLUME ["/data"] - -CMD ["start-tb.sh"] diff --git a/msa/tb/docker-tb/start-db.sh b/msa/tb/docker-tb/start-db.sh deleted file mode 100644 index 6dfddf8a7a..0000000000 --- a/msa/tb/docker-tb/start-db.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# Copyright © 2016-2022 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Do nothing \ No newline at end of file diff --git a/msa/tb/docker-tb/stop-db.sh b/msa/tb/docker-tb/stop-db.sh deleted file mode 100644 index 6dfddf8a7a..0000000000 --- a/msa/tb/docker-tb/stop-db.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# Copyright © 2016-2022 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Do nothing \ No newline at end of file diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index a1aa1c0e16..13f6ef3226 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -35,7 +35,6 @@ UTF-8 ${basedir}/../.. thingsboard - tb tb-postgres tb-cassandra /usr/share/${pkg.name} @@ -59,25 +58,6 @@ org.apache.maven.plugins maven-dependency-plugin - - copy-tb-deb - package - - copy - - - - - org.thingsboard - application - deb - deb - ${pkg.name}.deb - ${project.build.directory}/docker-tb - - - - copy-tb-postgres-deb package @@ -122,27 +102,7 @@ org.apache.maven.plugins maven-resources-plugin - - copy-docker-tb-config - process-resources - - copy-resources - - - ${project.build.directory}/docker-tb - - - docker - true - - - docker-tb - true - - - - - + copy-docker-tb-postgres-config process-resources @@ -188,32 +148,6 @@ com.spotify dockerfile-maven-plugin - - build-docker-tb-image - pre-integration-test - - build - - - ${dockerfile.skip} - ${docker.repo}/${tb.docker.name} - true - false - ${project.build.directory}/docker-tb - - - - tag-docker-tb-image - pre-integration-test - - tag - - - ${dockerfile.skip} - ${docker.repo}/${tb.docker.name} - ${project.version} - - build-docker-tb-postgres-image pre-integration-test @@ -284,28 +218,6 @@ com.spotify dockerfile-maven-plugin - - push-latest-docker-tb-image - pre-integration-test - - push - - - latest - ${docker.repo}/${tb.docker.name} - - - - push-version-docker-tb-image - pre-integration-test - - push - - - ${project.version} - ${docker.repo}/${tb.docker.name} - - push-latest-docker-tb-postgres-image pre-integration-test @@ -368,48 +280,6 @@ org.codehaus.mojo exec-maven-plugin - - push-latest-docker-amd-arm-tb-images - pre-integration-test - - exec - - - docker - ${project.build.directory}/docker-tb - - buildx - build - -t - ${docker.repo}/${tb.docker.name}:latest - --platform=linux/amd64,linux/arm64 - -o - type=registry - . - - - - - push-version-docker-amd-arm-tb-images - pre-integration-test - - exec - - - docker - ${project.build.directory}/docker-tb - - buildx - build - -t - ${docker.repo}/${tb.docker.name}:${project.version} - --platform=linux/amd64,linux/arm64 - -o - type=registry - . - - - push-latest-docker-amd-arm-tb-postgres-images pre-integration-test diff --git a/pom.xml b/pom.xml index 78c4ae6fa2..9273e7bebf 100755 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,6 @@ 4.1.0 4.3.1.0 2.7.2 - 2.6.1 1.5.2 5.7.2 2.6.0 @@ -1659,11 +1658,6 @@ bcpkix-jdk15on ${bouncycastle.version} - - org.hsqldb - hsqldb - ${hsqldb.version} - org.testcontainers postgresql From 2fb13de456bdbdf43cce4b23c8eff4bd5dab87ec Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 08:47:19 +0300 Subject: [PATCH 237/262] Backport of the permance improvements from PE --- .../impl/BaseEntityImportService.java | 58 ++++++++++++------- .../impl/DashboardImportService.java | 6 +- .../impl/RuleChainImportService.java | 6 +- .../DefaultEntitiesVersionControlService.java | 15 ++--- .../sync/vc/data/EntitiesImportCtx.java | 17 ++++-- .../service/sync/vc/data/ReimportTask.java | 28 +++++++++ .../data/sync/ie/EntityImportSettings.java | 2 - .../relation/SqlRelationInsertRepository.java | 57 ++++++++---------- 8 files changed, 116 insertions(+), 73 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index df6b770d1f..8ef0ce8fc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -60,6 +60,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -280,40 +281,31 @@ public abstract class BaseEntityImportService getInternalIdByUuid(UUID externalUuid) { + public Optional getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set hints) { if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty(); for (EntityType entityType : EntityType.values()) { - EntityId externalId; - try { - externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); - } catch (Exception e) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (!externalIdOpt.isPresent()) { continue; } - EntityId internalId = ctx.getInternalId(externalId); + EntityId internalId = ctx.getInternalId(externalIdOpt.get()); if (internalId != null) { return Optional.of(internalId); } } - for (EntityType entityType : EntityType.values()) { - EntityId externalId; - try { - externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); - } catch (Exception e) { - continue; + if (fetchAllUUIDs) { + for (EntityType entityType : hints) { + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; } - - EntityId internalId = getInternalId(externalId, false); - if (internalId != null) { - return Optional.of(internalId); - } else if (ctx.isResetExternalIdsOfAnotherTenant()) { - try { - if (exportableEntitiesService.findEntityById(externalId) != null) { - return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, EntityId.NULL_UUID)); - } - } catch (Exception ignored) { + for (EntityType entityType : EntityType.values()) { + if (hints.contains(entityType)) { + continue; } + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; } } @@ -321,6 +313,28 @@ public abstract class BaseEntityImportService lookupInDb(UUID externalUuid, EntityType entityType) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) { + return Optional.empty(); + } + EntityId internalId = getInternalId(externalIdOpt.get(), false); + if (internalId != null) { + return Optional.of(internalId); + } else { + ctx.registerNotFound(externalIdOpt.get()); + } + return Optional.empty(); + } + + private Optional buildEntityId(EntityType entityType, UUID externalUuid) { + try { + return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid)); + } catch (Exception e) { + return Optional.empty(); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 24238c78e5..5c7e686c4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -37,8 +37,10 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -49,6 +51,8 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DashboardImportService extends BaseEntityImportService> { + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET)); + private final DashboardService dashboardService; @@ -78,7 +82,7 @@ public class DashboardImportService extends BaseEntityImportService { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) .map(entityId -> entityId.getId().toString()).orElse(uuid); })); ((ObjectNode) entityAlias).set(field, newFieldValue); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 4ffcb7a41c..d32f9589f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -35,7 +35,9 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Optional; import java.util.UUID; @@ -44,6 +46,8 @@ import java.util.UUID; @RequiredArgsConstructor public class RuleChainImportService extends BaseEntityImportService { + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); + private final RuleChainService ruleChainService; @Override @@ -70,7 +74,7 @@ public class RuleChainImportService extends BaseEntityImportService { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) .map(entityId -> entityId.getId().toString()) .orElse(uuid); }); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 69be12d580..61bce45a72 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx; +import org.thingsboard.server.service.sync.vc.data.ReimportTask; import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -282,8 +283,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityType entityType : entityTypes) { log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); sw.startNew("Entities " + entityType.name()); - EntityImportSettings settings = getEntityImportSettings(request, entityType); - ctx.setSettings(settings); + ctx.setSettings(getEntityImportSettings(request, entityType)); importEntities(ctx, entityType); } @@ -337,7 +337,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new LoadEntityException(entityData, e); } if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings()); + ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(entityData, ctx.getSettings())); continue; } @@ -351,11 +351,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"rawtypes", "unchecked"}) private void reimport(EntitiesImportCtx ctx) { - ctx.getToReimport().forEach((externalId, importSettings) -> { + ctx.setFetchAllUUIDs(true); + ctx.getToReimport().forEach((externalId, task) -> { try { - EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), ctx.getVersionId(), externalId).get(); - importSettings.setResetExternalIdsOfAnotherTenant(true); - ctx.setSettings(importSettings); + EntityExportData entityData = task.getData(); + var settings = task.getSettings(); + ctx.setSettings(settings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); ctx.registerResult(externalId.getEntityType(), importResult.getOldEntity() == null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 497d2b9fab..74d09a5c3b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -29,6 +29,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -43,14 +44,15 @@ public class EntitiesImportCtx { private final Map results = new HashMap<>(); private final Map> importedEntities = new HashMap<>(); - private final Map toReimport = new HashMap<>(); + private final Map toReimport = new HashMap<>(); private final List referenceCallbacks = new ArrayList<>(); private final List eventCallbacks = new ArrayList<>(); - private final Map externalToInternalIdMap = new HashMap<>(); + private final Set notFoundIds = new HashSet<>(); private final Set relations = new LinkedHashSet<>(); + private boolean fetchAllUUIDs = false; private EntityImportSettings settings; public EntitiesImportCtx(SecurityUser user, String versionId) { @@ -83,10 +85,6 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } - public boolean isResetExternalIdsOfAnotherTenant() { - return getSettings().isResetExternalIdsOfAnotherTenant(); - } - public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); @@ -128,5 +126,12 @@ public class EntitiesImportCtx { } } + public void registerNotFound(EntityId externalId) { + notFoundIds.add(externalId); + } + + public boolean isNotFound(EntityId externalId) { + return notFoundIds.contains(externalId); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java new file mode 100644 index 0000000000..97432adbb8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; + +@Data +public class ReimportTask { + + private final EntityExportData data; + private final EntityImportSettings settings; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 6db4655c59..addc8c35f2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -30,6 +30,4 @@ public class EntityImportSettings { private boolean saveAttributes; private boolean saveCredentials; - // internal - private boolean resetExternalIdsOfAnotherTenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index b0ce419cac..cbc4ca1e02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -19,10 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; -import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; -import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.model.sql.RelationEntity; @@ -52,9 +49,6 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { @Autowired protected JdbcTemplate jdbcTemplate; - @Autowired - private TransactionTemplate transactionTemplate; - protected Query getQuery(RelationEntity entity, String query) { Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); if (entity.getAdditionalInfo() == null) { @@ -78,36 +72,31 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { @Override public void saveOrUpdate(List entities) { - transactionTemplate.execute(new TransactionCallbackWithoutResult() { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { @Override - protected void doInTransactionWithoutResult(TransactionStatus status) { - jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - RelationEntity relation = entities.get(i); - ps.setObject(1, relation.getFromId()); - ps.setString(2, relation.getFromType()); - ps.setObject(3, relation.getToId()); - ps.setString(4, relation.getToType()); - - ps.setString(5, relation.getRelationTypeGroup()); - ps.setString(6, relation.getRelationType()); - - if (relation.getAdditionalInfo() == null) { - ps.setString(7, null); - ps.setString(8, null); - } else { - String json = JacksonUtil.toString(relation.getAdditionalInfo()); - ps.setString(7, json); - ps.setString(8, json); - } - } + public void setValues(PreparedStatement ps, int i) throws SQLException { + RelationEntity relation = entities.get(i); + ps.setObject(1, relation.getFromId()); + ps.setString(2, relation.getFromType()); + ps.setObject(3, relation.getToId()); + ps.setString(4, relation.getToType()); + + ps.setString(5, relation.getRelationTypeGroup()); + ps.setString(6, relation.getRelationType()); + + if (relation.getAdditionalInfo() == null) { + ps.setString(7, null); + ps.setString(8, null); + } else { + String json = JacksonUtil.toString(relation.getAdditionalInfo()); + ps.setString(7, json); + ps.setString(8, json); + } + } - @Override - public int getBatchSize() { - return entities.size(); - } - }); + @Override + public int getBatchSize() { + return entities.size(); } }); } From 76525766622503e3fc6ea6e5c78eaead266e83ee Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 16 Jun 2022 11:04:12 +0300 Subject: [PATCH 238/262] Postgresql repo from os-release $VERSION_CODENAME for dockerfile. Disable docker cache temporary to avoid cached releases collission --- msa/tb/docker-cassandra/Dockerfile | 2 +- msa/tb/docker-postgres/Dockerfile | 2 +- msa/tb/pom.xml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index 5b22efd636..f61f070486 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -18,7 +18,7 @@ FROM thingsboard/openjdk11 RUN apt-get update RUN apt-get install -y curl nmap procps gnupg2 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main' | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null RUN curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo 'deb http://downloads.apache.org/cassandra/debian 40x main' | tee --append /etc/apt/sources.list.d/cassandra.list > /dev/null RUN curl -L https://downloads.apache.org/cassandra/KEYS | apt-key add - diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index 720f770007..d70bc29b91 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -18,7 +18,7 @@ FROM thingsboard/openjdk11 RUN apt-get update RUN apt-get install -y curl gnupg2 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main' | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null RUN curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - ENV PG_MAJOR 12 RUN apt-get update diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index 13f6ef3226..a69b358e64 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -160,6 +160,7 @@ true false ${project.build.directory}/docker-postgres + true @@ -186,6 +187,7 @@ true false ${project.build.directory}/docker-cassandra + true From 88fcff5556f1516ccf5a144c271a4035c3657e87 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 16 Jun 2022 11:10:49 +0300 Subject: [PATCH 239/262] Set external ids in the config of rule chains and dashboards on export; add JsonIgnoreProperties in export data --- .../impl/DashboardExportService.java | 22 ++++++++++-- .../impl/DefaultEntityExportService.java | 35 ++++++++++++++++--- .../impl/RuleChainExportService.java | 24 ++++++++++++- .../impl/BaseEntityImportService.java | 7 ---- .../DefaultEntitiesVersionControlService.java | 1 - .../sync/vc/data/EntitiesImportCtx.java | 4 --- .../server/common/data/Dashboard.java | 11 ++++++ .../common/data/sync/ie/DeviceExportData.java | 2 ++ .../common/data/sync/ie/EntityExportData.java | 1 + .../data/sync/ie/EntityImportSettings.java | 3 -- .../data/sync/ie/RuleChainExportData.java | 2 ++ 11 files changed, 90 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 1064962082..391606be63 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -15,18 +15,23 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.utils.RegexUtils; +import java.util.ArrayList; import java.util.Set; +import java.util.UUID; @Service @TbCoreComponent @@ -39,6 +44,19 @@ public class DashboardExportService extends BaseEntityExportService fields = Lists.newArrayList(entityAlias.fieldNames()); + for (String field : fields) { + if (field.equals("id")) continue; + JsonNode oldFieldValue = entityAlias.get(field); + JsonNode newFieldValue = JacksonUtil.toJsonNode(RegexUtils.replace(oldFieldValue.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); + })); + ((ObjectNode) entityAlias).set(field, newFieldValue); + } + } + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 4f748ba9d8..7e236370df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -24,27 +24,25 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -150,6 +148,35 @@ public class DefaultEntityExportService ctx, UUID internalUuid) { + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = ctx.getExternalId(internalId); + if (externalId != null) { + return externalId.getId(); + } + } + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = exportableEntitiesService.getExternalIdByInternal(internalId); + if (externalId != null) { + ctx.putExternalId(internalId, externalId); + return externalId.getId(); + } + } + return internalUuid; + } + protected D newExportData() { return (D) new EntityExportData(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index 9af29cceba..e4a0cfd729 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -15,19 +15,26 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; +import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.utils.RegexUtils; +import java.util.Collections; +import java.util.Optional; import java.util.Set; +import java.util.UUID; @Service @TbCoreComponent @@ -38,7 +45,22 @@ public class RuleChainExportService extends BaseEntityExportService ctx, RuleChain ruleChain, RuleChainExportData exportData) { - exportData.setMetaData(ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId())); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId()); + Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()) + .forEach(ruleNode -> { + ruleNode.setRuleChainId(null); + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); + String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); + }); + ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); + ruleNode.setConfiguration(ruleNodeConfig); + }); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(getExternalIdOrElseInternal(ctx, ruleChainConnectionInfo.getTargetRuleChainId())); + }); + exportData.setMetaData(metaData); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index df6b770d1f..8b203075ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -307,13 +307,6 @@ public abstract class BaseEntityImportService { try { EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), ctx.getVersionId(), externalId).get(); - importSettings.setResetExternalIdsOfAnotherTenant(true); ctx.setSettings(importSettings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 497d2b9fab..842e3feb67 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -83,10 +83,6 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } - public boolean isResetExternalIdsOfAnotherTenant() { - return getSettings().isResetExternalIdsOfAnotherTenant(); - } - public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index b9f2bf1fee..b1ca86de10 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -15,13 +15,17 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; +import java.util.Optional; + @EqualsAndHashCode(callSuper = true) public class Dashboard extends DashboardInfo implements ExportableEntity { @@ -62,6 +66,13 @@ public class Dashboard extends DashboardInfo implements ExportableEntity config.get("entityAliases")) + .filter(JsonNode::isObject).orElse(null); + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index 8edbc46f97..27d16899d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.EqualsAndHashCode; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; public class DeviceExportData extends EntityExportData { @JsonProperty(index = 3) + @JsonIgnoreProperties({"id", "deviceId", "createdTime"}) private DeviceCredentials credentials; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 5283894739..afe6c925c7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -55,6 +55,7 @@ public class EntityExportData> { .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); @JsonProperty(index = 2) + @JsonIgnoreProperties({"tenantId", "createdTime"}) @JsonTbEntity private E entity; @JsonProperty(index = 1) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 6db4655c59..9b95b8eca2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -29,7 +29,4 @@ public class EntityImportSettings { private boolean updateRelations; private boolean saveAttributes; private boolean saveCredentials; - - // internal - private boolean resetExternalIdsOfAnotherTenant; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java index e9f4086101..5c37aa35b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.ie; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.EqualsAndHashCode; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; public class RuleChainExportData extends EntityExportData { @JsonProperty(index = 3) + @JsonIgnoreProperties("ruleChainId") private RuleChainMetaData metaData; } From 15c1e215d44cefed77ffaa4567f977fa1fa2a66e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 12:09:17 +0300 Subject: [PATCH 240/262] Improve performance of relation and rule chain service --- .../processing/AbstractConsumerService.java | 2 +- .../impl/BaseEntityImportService.java | 13 ++++- .../service/DefaultTransportService.java | 2 +- .../dao/relation/BaseRelationService.java | 50 +++++++------------ .../server/dao/relation/RelationDao.java | 8 ++- .../server/dao/rule/BaseRuleChainService.java | 29 ++++++++--- .../server/dao/rule/RuleNodeDao.java | 3 ++ .../dao/sql/relation/JpaRelationDao.java | 48 ++++++++++++++---- .../dao/sql/relation/RelationRepository.java | 11 ++++ .../server/dao/sql/rule/JpaRuleNodeDao.java | 9 +++- .../dao/sql/rule/RuleNodeRepository.java | 2 + 11 files changed, 121 insertions(+), 56 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index f1be88a44d..0c130c9f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -160,7 +160,7 @@ public abstract class AbstractConsumerService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { +// TbStopWatch sw = TbStopWatch.create("find"); EntityImportResult importResult = new EntityImportResult<>(); IdProvider idProvider = new IdProvider(ctx, importResult); @@ -100,16 +103,22 @@ public abstract class BaseEntityImportService inboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelations.addAll(relationDao.findAllByTo(tenantId, entityId, typeGroup)); - } - - List outboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelations.addAll(relationDao.findAllByFrom(tenantId, entityId, typeGroup)); - } + List inboundRelations = new ArrayList<>(relationDao.findAllByTo(tenantId, entityId)); + List outboundRelations = new ArrayList<>(relationDao.findAllByFrom(tenantId, entityId)); - for (EntityRelation relation : inboundRelations) { - delete(tenantId, relation, true); - } + if (!inboundRelations.isEmpty()) { + try { + relationDao.deleteInboundRelations(tenantId, entityId); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", inboundRelations, e); + } - for (EntityRelation relation : outboundRelations) { - delete(tenantId, relation, false); + for (EntityRelation relation : inboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - relationDao.deleteOutboundRelations(tenantId, entityId); + if (!outboundRelations.isEmpty()) { + relationDao.deleteOutboundRelations(tenantId, entityId); + for (EntityRelation relation : outboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } + } } @Override @@ -269,18 +267,6 @@ public class BaseRelationService implements RelationService { } } - boolean delete(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - if (deleteFromDb) { - try { - return relationDao.deleteRelation(tenantId, relation); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", relation, e); - } - } - return false; - } - @Override public List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { validate(from); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 9ed3a5672f..180996a6e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -32,10 +32,14 @@ public interface RelationDao { List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); + List findAllByFrom(TenantId tenantId, EntityId from); + List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + List findAllByTo(TenantId tenantId, EntityId to); + List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); @@ -56,7 +60,9 @@ public interface RelationDao { ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean deleteOutboundRelations(TenantId tenantId, EntityId entity); + void deleteOutboundRelations(TenantId tenantId, EntityId entity); + + void deleteInboundRelations(TenantId tenantId, EntityId entity); ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 03b3886cfc..f785eae0b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -141,6 +141,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List nodes = ruleChainMetaData.getNodes(); List toAddOrUpdate = new ArrayList<>(); List toDelete = new ArrayList<>(); + List relations = new ArrayList<>(); Map ruleNodeIndexMap = new HashMap<>(); if (nodes != null) { @@ -171,15 +172,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (RuleNode node : toAddOrUpdate) { node.setRuleChainId(ruleChain.getId()); RuleNode savedNode = ruleNodeDao.save(tenantId, node); - createRelation(tenantId, new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), + relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); int index = nodes.indexOf(node); nodes.set(index, savedNode); ruleNodeIndexMap.put(savedNode.getId(), index); } } - for (RuleNode node : toDelete) { - deleteRuleNode(tenantId, node.getId()); + if (!toDelete.isEmpty()) { + deleteRuleNodes(tenantId, toDelete); } RuleNodeId firstRuleNodeId = null; if (nodes != null) { @@ -196,7 +197,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC EntityId from = nodes.get(nodeConnection.getFromIndex()).getId(); EntityId to = nodes.get(nodeConnection.getToIndex()).getId(); String type = nodeConnection.getType(); - createRelation(tenantId, new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); + relations.add(new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); } } if (ruleChainMetaData.getRuleChainConnections() != null) { @@ -222,7 +223,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleChainToRuleNode.setTo(targetNode.getId()); sourceRuleChainToRuleNode.setType(EntityRelation.CONTAINS_TYPE); sourceRuleChainToRuleNode.setTypeGroup(RelationTypeGroup.RULE_CHAIN); - relationService.saveRelation(tenantId, sourceRuleChainToRuleNode); + relations.add(sourceRuleChainToRuleNode); EntityRelation sourceRuleNodeToTargetRuleNode = new EntityRelation(); EntityId from = nodes.get(nodeToRuleChainConnection.getFromIndex()).getId(); @@ -230,11 +231,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleNodeToTargetRuleNode.setTo(targetNode.getId()); sourceRuleNodeToTargetRuleNode.setType(nodeToRuleChainConnection.getType()); sourceRuleNodeToTargetRuleNode.setTypeGroup(RelationTypeGroup.RULE_NODE); - relationService.saveRelation(tenantId, sourceRuleNodeToTargetRuleNode); - } + relations.add(sourceRuleNodeToTargetRuleNode); + } } } + if (!relations.isEmpty()) { + relationService.saveRelations(tenantId, relations); + } + return RuleChainUpdateResult.successful(updatedRuleNodes); } @@ -710,6 +715,14 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC deleteRuleNodes(tenantId, ruleChainId); } + private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { + List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + for (var node : ruleNodes) { + deleteEntityRelations(tenantId, node.getId()); + } + ruleNodeDao.deleteByIdIn(ruleNodeIds); + } + @Override public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index b4f087b98a..bd3efcd205 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,4 +32,6 @@ public interface RuleNodeDao extends Dao { List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search); PageData findAllRuleNodesByType(String type, PageLink pageLink); + + void deleteByIdIn(List ruleNodeIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 550197c924..5b8a1a6e5d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -33,6 +33,8 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -44,6 +46,12 @@ import java.util.stream.Collectors; @Component public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { + private static final List ALL_TYPE_GROUP_NAMES = new ArrayList<>(); + + static { + Arrays.stream(RelationTypeGroup.values()).map(RelationTypeGroup::name).forEach(ALL_TYPE_GROUP_NAMES::add); + } + @Autowired private RelationRepository relationRepository; @@ -59,6 +67,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByFrom(TenantId tenantId, EntityId from) { + return DaoUtil.convertDataList( + relationRepository.findAllByFromIdAndFromTypeAndRelationTypeGroupIn( + from.getId(), + from.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -78,6 +95,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByTo(TenantId tenantId, EntityId to) { + return DaoUtil.convertDataList( + relationRepository.findAllByToIdAndToTypeAndRelationTypeGroupIn( + to.getId(), + to.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -164,19 +190,21 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public boolean deleteOutboundRelations(TenantId tenantId, EntityId entity) { - boolean relationExistsBeforeDelete = false; + public void deleteOutboundRelations(TenantId tenantId, EntityId entity) { try { - relationExistsBeforeDelete = relationRepository - .findAllByFromIdAndFromType(entity.getId(), entity.getEntityType().name()) - .size() > 0; - if (relationExistsBeforeDelete) { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } + relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", entity, e); + } + } + + @Override + public void deleteInboundRelations(TenantId tenantId, EntityId entity) { + try { + relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), ALL_TYPE_GROUP_NAMES); } catch (ConcurrencyFailureException e) { log.debug("Concurrency exception while deleting relations [{}]", entity, e); } - return relationExistsBeforeDelete; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index bddd344672..1f776fd44d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -35,6 +35,10 @@ public interface RelationRepository String fromType, String relationTypeGroup); + List findAllByFromIdAndFromTypeAndRelationTypeGroupIn(UUID fromId, + String fromType, + List relationTypeGroups); + List findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup(UUID fromId, String fromType, String relationType, @@ -44,6 +48,10 @@ public interface RelationRepository String toType, String relationTypeGroup); + List findAllByToIdAndToTypeAndRelationTypeGroupIn(UUID toId, + String toType, + List relationTypeGroups); + List findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup(UUID toId, String toType, String relationType, @@ -66,4 +74,7 @@ public interface RelationRepository @Transactional void deleteByFromIdAndFromType(UUID fromId, String fromType); + @Transactional + void deleteByToIdAndToTypeAndRelationTypeGroupIn(UUID fromId, String fromType, List relationTypeGroups); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 1c8603e769..eeff3f8c79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -32,6 +33,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Component @@ -64,6 +66,11 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao ruleNodeIds) { + ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); + } + @Override public EntityType getEntityType() { return EntityType.RULE_NODE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index f584ff2b02..9fee64824b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -39,4 +39,6 @@ public interface RuleNodeRepository extends JpaRepository @Param("searchText") String searchText, Pageable pageable); + void deleteByIdIn(List ids); + } From 3581172e8fc261b6d0c18d0c7acaa9ce3ce2a1b3 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 16 Jun 2022 12:32:12 +0300 Subject: [PATCH 241/262] Tests for export/import api --- .../sync/ie/BaseExportImportServiceTest.java | 373 ++++++++++++++ .../sync/ie/ExportImportServiceSqlTest.java | 487 ++++++++++++++++++ 2 files changed, 860 insertions(+) create mode 100644 application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java new file mode 100644 index 0000000000..06f8bcd281 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -0,0 +1,373 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +public abstract class BaseExportImportServiceTest extends AbstractControllerTest { + + @Autowired + protected EntitiesExportImportService exportImportService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected OtaPackageService otaPackageService; + @Autowired + protected DeviceProfileService deviceProfileService; + @Autowired + protected AssetService assetService; + @Autowired + protected CustomerService customerService; + @Autowired + protected RuleChainService ruleChainService; + @Autowired + protected DashboardService dashboardService; + @Autowired + protected RelationService relationService; + @Autowired + protected TenantService tenantService; + + protected TenantId tenantId1; + protected User tenantAdmin1; + + protected TenantId tenantId2; + protected User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } + + protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName(name); + device.setLabel("lbl"); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + device.setDeviceData(deviceData); + return deviceService.saveDevice(device); + } + + protected OtaPackage createOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type) { + OtaPackage otaPackage = new OtaPackage(); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfileId); + otaPackage.setType(type); + otaPackage.setTitle("My " + type); + otaPackage.setVersion("v1.0"); + otaPackage.setFileName("filename.txt"); + otaPackage.setContentType("text/plain"); + otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + otaPackage.setDataSize(1L); + otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); + return otaPackageService.saveOtaPackage(otaPackage); + } + + protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, RuleChainId defaultRuleChainId, DashboardId defaultDashboardId, String name) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setName(name); + deviceProfile.setDescription("dscrptn"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + deviceProfile.setDefaultRuleChainId(defaultRuleChainId); + deviceProfile.setDefaultDashboardId(defaultDashboardId); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + protected void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCustomerId(customerId); + asset.setType(type); + asset.setName(name); + asset.setLabel("lbl"); + asset.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return assetService.saveAsset(asset); + } + + protected void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + + protected Customer createCustomer(TenantId tenantId, String name) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(name); + customer.setCountry("ua"); + customer.setAddress("abb"); + customer.setEmail("ccc@aa.org"); + customer.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return customerService.saveCustomer(customer); + } + + protected void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setTitle(name); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + dashboard.setImage("abvregewrg"); + dashboard.setMobileHide(true); + dashboard = dashboardService.saveDashboard(dashboard); + if (customerId != null) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + return dashboardService.findDashboardById(tenantId, dashboard.getId()); + } + return dashboard; + } + + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + + protected RuleChain createRuleChain(TenantId tenantId, String name) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + protected EntityRelation createRelation(EntityId from, EntityId to) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(from); + relation.setTo(to); + relation.setType(EntityRelation.MANAGES_TYPE); + relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + return relation; + } + + protected & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + } + } + + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { + return exportEntity(user, entityId, EntityExportSettings.builder() + .exportCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId, EntityExportSettings exportSettings) throws Exception { + return exportImportService.exportEntity(new SimpleEntitiesExportCtx(getSecurityUser(user), null, null, exportSettings), entityId); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { + return importEntity(user, exportData, EntityImportSettings.builder() + .saveCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData, EntityImportSettings importSettings) throws Exception { + EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings); + exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class); + EntityImportResult importResult = exportImportService.importEntity(ctx, exportData); + exportImportService.saveReferencesAndRelations(ctx); + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + return importResult; + } + + protected SecurityUser getSecurityUser(User user) { + return new SecurityUser(user, true, new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail())); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java new file mode 100644 index 0000000000..63029023f3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -0,0 +1,487 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.Streams; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.ota.OtaPackageStateService; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; + +@DaoSqlTest +public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { + + @Autowired + private DeviceCredentialsService deviceCredentialsService; + @SpyBean + private EntityActionService entityActionService; + @SpyBean + private TbClusterService clusterService; + @SpyBean + private OtaPackageStateService otaPackageStateService; + + @Test + public void testExportImportAsset_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, asset, tenantId2, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, asset, tenantId1, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant_withCustomer() throws Exception { + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + Asset importedAsset = importEntity(tenantAdmin1, this.exportEntity(tenantAdmin1, asset.getId())).getSavedEntity(); + assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); + } + + + @Test + public void testExportImportCustomer_betweenTenants() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, customer, tenantId2, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + @Test + public void testExportImportCustomer_sameTenant() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, customer, tenantId1, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + + @Test + public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile of tenant 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData profileExportData = exportEntity(tenantAdmin1, deviceProfile.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + DeviceCredentials exportedCredentials = ((DeviceExportData) deviceExportData).getCredentials(); + exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); + + EntityImportResult profileImportResult = importEntity(tenantAdmin2, profileExportData); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult.getSavedEntity()); + checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); + + EntityImportResult deviceImportResult = importEntity(tenantAdmin2, deviceExportData); + Device importedDevice = deviceImportResult.getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult.getSavedEntity()); + checkImportedDeviceData(device, importedDevice); + + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getId()).isNotEqualTo(credentials.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(exportedCredentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + } + + @Test + public void testExportImportDevice_sameTenant() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile v1.0"); + OtaPackage firmware = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.FIRMWARE); + OtaPackage software = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.SOFTWARE); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + device.setFirmwareId(firmware.getId()); + device.setSoftwareId(software.getId()); + device = deviceService.saveDevice(device); + + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, deviceExportData); + Device importedDevice = importResult.getSavedEntity(); + + checkImportedEntity(tenantId1, device, tenantId1, importResult.getSavedEntity()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); + assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + assertThat(importedDevice.getFirmwareId()).isEqualTo(firmware.getId()); + assertThat(importedDevice.getSoftwareId()).isEqualTo(software.getId()); + } + + + @Test + public void testExportImportDashboard_betweenTenants() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_sameTenant() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_betweenTenants_withCustomer_updated() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Dashboard importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + EntityExportData customerExportData = exportEntity(tenantAdmin1, customer.getId()); + dashboardService.assignDashboardToCustomer(tenantId1, dashboard.getId(), customer.getId()); + exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Customer importedCustomer = importEntity(tenantAdmin2, customerExportData).getSavedEntity(); + importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } + + @Test + public void testExportImportDashboard_betweenTenants_withEntityAliases() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + asset1.getId().toString() + "\",\n" + + "\t\t\t\t\"" + asset2.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + dashboard = dashboardService.saveDashboard(dashboard); + + EntityExportData asset1ExportData = exportEntity(tenantAdmin1, asset1.getId()); + EntityExportData asset2ExportData = exportEntity(tenantAdmin1, asset2.getId()); + EntityExportData dashboardExportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Asset importedAsset1 = importEntity(tenantAdmin2, asset1ExportData).getSavedEntity(); + Asset importedAsset2 = importEntity(tenantAdmin2, asset2ExportData).getSavedEntity(); + Dashboard importedDashboard = importEntity(tenantAdmin2, dashboardExportData).getSavedEntity(); + + Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .map(JsonNode::asText).collect(Collectors.toSet()); + assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); + assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); + } + + + @Test + public void testExportImportRuleChain_betweenTenants() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + @Test + public void testExportImportRuleChain_sameTenant() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + + @Test + public void testExportImportWithInboundRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + + assertThat(deviceExportData.getRelations()).size().isOne(); + assertThat(deviceExportData.getRelations().get(0)).matches(entityRelation -> { + return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); + }); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + deviceExportData.getEntity().setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device1 = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(assetExportData.getRelations()).size().isOne(); + + Device device2 = createDevice(tenantId1, null, null, "Device 2"); + EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); + + importEntity(tenantAdmin1, assetExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + @Test + public void textExportImportWithRelations_sameTenant_removeExisting() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(deviceExportData.getRelations()).size().isOne(); + + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); + + importEntity(tenantAdmin1, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByTo(TenantId.SYS_TENANT_ID, device.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + + @Test + public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { + DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); + EntityExportData deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId()); + + assertThatThrownBy(() -> { + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(false) + .build()); + }).hasMessageContaining("default device profile is present"); + + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(true) + .build()); + checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); + } + + + @Test + public void testEntityEventsOnImport() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + + Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), + ruleChain.getId(), dashboard.getId(), deviceProfile.getId()) + .map(entityId -> { + try { + return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() + .exportCredentials(false) + .build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toMap(EntityExportData::getEntityType, d -> d)); + + Customer importedCustomer = (Customer) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.CUSTOMER)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.ADDED), isNull()); + importEntity(tenantAdmin2, entitiesExportData.get(EntityType.CUSTOMER)); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + Asset importedAsset = (Asset) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.ADDED), isNull()); + importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.RULE_CHAIN)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + + Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DASHBOARD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), + any(), eq(ActionType.ADDED), isNull()); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE_PROFILE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); + + Device importedDevice = (Device) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceUpdated(eq(importedDevice), isNull()); + importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)); + verify(clusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + } + +} From c0e84e22426d94f1979a44f158e14700ab1e5ab8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 12:33:06 +0300 Subject: [PATCH 242/262] Fix license headers --- .../service/sync/ie/importing/impl/BaseEntityImportService.java | 2 +- .../thingsboard/server/dao/relation/BaseRelationService.java | 2 +- .../org/thingsboard/server/dao/rule/BaseRuleChainService.java | 2 +- .../org/thingsboard/server/dao/sql/relation/JpaRelationDao.java | 2 +- .../org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index cfe4dfee9c..0567614c6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index a18bd6ab63..7698adfda2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index f785eae0b7..7660e489fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 5b8a1a6e5d..b5d6e747d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index eeff3f8c79..47020657e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 4195d48c3bff5814a9bbc52ea081b85cdb161cc7 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 16 Jun 2022 12:39:39 +0300 Subject: [PATCH 243/262] Updated and optimized docker build files. --- msa/js-executor/docker/Dockerfile | 16 +++--- msa/js-executor/pom.xml | 2 +- msa/tb-node/docker/Dockerfile | 7 +-- msa/tb/docker-cassandra/Dockerfile | 80 ++++++++++++--------------- msa/tb/docker-postgres/Dockerfile | 61 +++++++++----------- msa/transport/coap/docker/Dockerfile | 2 +- msa/transport/http/docker/Dockerfile | 2 +- msa/transport/lwm2m/docker/Dockerfile | 2 +- msa/transport/mqtt/docker/Dockerfile | 2 +- msa/transport/snmp/docker/Dockerfile | 2 +- msa/web-ui/docker/Dockerfile | 16 +++--- msa/web-ui/pom.xml | 2 +- ui-ngx/pom.xml | 2 +- 13 files changed, 88 insertions(+), 108 deletions(-) diff --git a/msa/js-executor/docker/Dockerfile b/msa/js-executor/docker/Dockerfile index 0934b4b1a1..620d712ad1 100644 --- a/msa/js-executor/docker/Dockerfile +++ b/msa/js-executor/docker/Dockerfile @@ -14,16 +14,13 @@ # limitations under the License. # -FROM node:16.13.1-bullseye-slim - -COPY start-js-executor.sh /tmp/ - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-js-executor.sh /usr/bin +FROM node:16.15.1-bullseye-slim ENV NODE_ENV production ENV DOCKER_MODE true +COPY start-js-executor.sh /tmp/ + WORKDIR ${pkg.installFolder} COPY ["src/package.json", "src/yarn.lock", "./"] @@ -34,9 +31,10 @@ COPY src/api ./api COPY src/queue ./queue COPY src/server.js ./ -RUN chown -R node:node ${pkg.installFolder} - -RUN yarn install --production && yarn cache clean --all +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-js-executor.sh /usr/bin \ + && chown -R node:node ${pkg.installFolder} \ + && yarn install --production && yarn cache clean --all USER node diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index a910e4759f..8e376e1e15 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -71,7 +71,7 @@ install-node-and-yarn - v16.13.1 + v16.15.1 v1.22.17 diff --git a/msa/tb-node/docker/Dockerfile b/msa/tb-node/docker/Dockerfile index 1881351b92..cbfbebf86c 100644 --- a/msa/tb-node/docker/Dockerfile +++ b/msa/tb-node/docker/Dockerfile @@ -14,13 +14,12 @@ # limitations under the License. # -FROM thingsboard/openjdk11 - -RUN echo 'networkaddress.cache.ttl=60' >> /etc/java-11-openjdk/security/java.security +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-node.sh ${pkg.name}.deb /tmp/ -RUN chmod a+x /tmp/*.sh \ +RUN echo 'networkaddress.cache.ttl=60' >> /etc/java-11-openjdk/security/java.security \ + && chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-node.sh /usr/bin && \ (yes | dpkg -i /tmp/${pkg.name}.deb) && \ rm /tmp/${pkg.name}.deb && \ diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index f61f070486..f3fa3146c0 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -14,37 +14,9 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim -RUN apt-get update -RUN apt-get install -y curl nmap procps gnupg2 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null -RUN curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo 'deb http://downloads.apache.org/cassandra/debian 40x main' | tee --append /etc/apt/sources.list.d/cassandra.list > /dev/null -RUN curl -L https://downloads.apache.org/cassandra/KEYS | apt-key add - ENV PG_MAJOR=12 -RUN apt-get update -RUN apt-get install -y cassandra cassandra-tools postgresql-12 -RUN update-rc.d cassandra disable -RUN update-rc.d postgresql disable -RUN sed -i.old '/ulimit/d' /etc/init.d/cassandra - -COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-tb.sh /usr/bin \ - && mv /tmp/upgrade-tb.sh /usr/bin \ - && mv /tmp/install-tb.sh /usr/bin \ - && mv /tmp/start-db.sh /usr/bin \ - && mv /tmp/stop-db.sh /usr/bin - -RUN dpkg -i /tmp/${pkg.name}.deb -RUN rm /tmp/${pkg.name}.deb - -RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : - -RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \ - && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf ENV DATA_FOLDER=/data @@ -69,23 +41,43 @@ ENV PATH=$PATH:/usr/lib/postgresql/$PG_MAJOR/bin ENV PGLOG=/var/log/postgres ENV CASSANDRA_LOG=/var/log/cassandra -# postgres config -RUN mkdir -p $PGLOG -RUN chown -R ${pkg.user}:${pkg.user} $PGLOG -RUN chown -R ${pkg.user}:${pkg.user} /var/log/postgresql -RUN chown -R ${pkg.user}:${pkg.user} /var/run/postgresql +COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ +RUN apt-get update \ + && apt-get install -y curl nmap procps gnupg2 \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null \ + && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && echo 'deb http://downloads.apache.org/cassandra/debian 40x main' | tee --append /etc/apt/sources.list.d/cassandra.list > /dev/null \ + && curl -L https://downloads.apache.org/cassandra/KEYS | apt-key add - \ + && apt-get update \ + && apt-get install -y cassandra cassandra-tools postgresql-12 \ + && update-rc.d cassandra disable \ + && update-rc.d postgresql disable \ + && sed -i.old '/ulimit/d' /etc/init.d/cassandra \ + && chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb.sh /usr/bin \ + && mv /tmp/upgrade-tb.sh /usr/bin \ + && mv /tmp/install-tb.sh /usr/bin \ + && mv /tmp/start-db.sh /usr/bin \ + && mv /tmp/stop-db.sh /usr/bin \ + && dpkg -i /tmp/${pkg.name}.deb \ + && rm /tmp/${pkg.name}.deb \ + && systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : \ + && mv /tmp/logback.xml ${pkg.installFolder}/conf \ + && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf \ +# postgres config + && mkdir -p $PGLOG \ + && chown -R ${pkg.user}:${pkg.user} $PGLOG \ + && chown -R ${pkg.user}:${pkg.user} /var/log/postgresql \ + && chown -R ${pkg.user}:${pkg.user} /var/run/postgresql \ # cassandra config -RUN rm -rf /var/lib/cassandra -RUN chmod a+w /var/lib -RUN chown -R ${pkg.user}:${pkg.user} $CASSANDRA_LOG - - -RUN mkdir -p $DATA_FOLDER -RUN chown -R ${pkg.user}:${pkg.user} $DATA_FOLDER -RUN chown -R ${pkg.user}:${pkg.user} /var/log/${pkg.name} - -RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + && rm -rf /var/lib/cassandra \ + && chmod a+w /var/lib \ + && chown -R ${pkg.user}:${pkg.user} $CASSANDRA_LOG \ + && mkdir -p $DATA_FOLDER \ + && chown -R ${pkg.user}:${pkg.user} $DATA_FOLDER \ + && chown -R ${pkg.user}:${pkg.user} /var/log/${pkg.name} \ + && chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar USER ${pkg.user} diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index d70bc29b91..24c9200ea4 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -14,33 +14,9 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim -RUN apt-get update -RUN apt-get install -y curl gnupg2 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null -RUN curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - ENV PG_MAJOR 12 -RUN apt-get update -RUN apt-get install -y postgresql-12 -RUN update-rc.d postgresql disable - -COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-tb.sh /usr/bin \ - && mv /tmp/upgrade-tb.sh /usr/bin \ - && mv /tmp/install-tb.sh /usr/bin \ - && mv /tmp/start-db.sh /usr/bin \ - && mv /tmp/stop-db.sh /usr/bin - -RUN dpkg -i /tmp/${pkg.name}.deb -RUN rm /tmp/${pkg.name}.deb - -RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : - -RUN mv /tmp/logback.xml ${pkg.installFolder}/conf \ - && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf ENV DATA_FOLDER=/data @@ -55,18 +31,35 @@ ENV SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/thingsboard ENV SPRING_DATASOURCE_USERNAME=${pkg.user} ENV SPRING_DATASOURCE_PASSWORD=postgres - ENV PGLOG=/var/log/postgres -RUN mkdir -p $PGLOG -RUN chown -R ${pkg.user}:${pkg.user} $PGLOG -RUN chown -R ${pkg.user}:${pkg.user} /var/run/postgresql - -RUN mkdir -p /data -RUN chown -R ${pkg.user}:${pkg.user} /data +COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ -RUN chown -R ${pkg.user}:${pkg.user} /var/log/${pkg.name} -RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar +RUN apt-get update \ + && apt-get install -y curl gnupg2 \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null \ + && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && apt-get update \ + && apt-get install -y postgresql-12 \ + && update-rc.d postgresql disable \ + && chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb.sh /usr/bin \ + && mv /tmp/upgrade-tb.sh /usr/bin \ + && mv /tmp/install-tb.sh /usr/bin \ + && mv /tmp/start-db.sh /usr/bin \ + && mv /tmp/stop-db.sh /usr/bin \ + && dpkg -i /tmp/${pkg.name}.deb \ + && rm /tmp/${pkg.name}.deb \ + && systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : \ + && mv /tmp/logback.xml ${pkg.installFolder}/conf \ + && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf \ + && mkdir -p $PGLOG \ + && chown -R ${pkg.user}:${pkg.user} $PGLOG \ + && chown -R ${pkg.user}:${pkg.user} /var/run/postgresql \ + && mkdir -p /data \ + && chown -R ${pkg.user}:${pkg.user} /data \ + && chown -R ${pkg.user}:${pkg.user} /var/log/${pkg.name} \ + && chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar USER ${pkg.user} diff --git a/msa/transport/coap/docker/Dockerfile b/msa/transport/coap/docker/Dockerfile index 2d174a4827..8b1d5d7a0e 100644 --- a/msa/transport/coap/docker/Dockerfile +++ b/msa/transport/coap/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-coap-transport.sh ${pkg.name}.deb /tmp/ diff --git a/msa/transport/http/docker/Dockerfile b/msa/transport/http/docker/Dockerfile index 8e0075738e..d17afe7f54 100644 --- a/msa/transport/http/docker/Dockerfile +++ b/msa/transport/http/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-http-transport.sh ${pkg.name}.deb /tmp/ diff --git a/msa/transport/lwm2m/docker/Dockerfile b/msa/transport/lwm2m/docker/Dockerfile index f0c92419d0..d6fa010a1a 100644 --- a/msa/transport/lwm2m/docker/Dockerfile +++ b/msa/transport/lwm2m/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-lwm2m-transport.sh ${pkg.name}.deb /tmp/ diff --git a/msa/transport/mqtt/docker/Dockerfile b/msa/transport/mqtt/docker/Dockerfile index 378cd14dd5..d46af0138c 100644 --- a/msa/transport/mqtt/docker/Dockerfile +++ b/msa/transport/mqtt/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-mqtt-transport.sh ${pkg.name}.deb /tmp/ diff --git a/msa/transport/snmp/docker/Dockerfile b/msa/transport/snmp/docker/Dockerfile index 84d115f0d4..310d1b34e1 100644 --- a/msa/transport/snmp/docker/Dockerfile +++ b/msa/transport/snmp/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-snmp-transport.sh ${pkg.name}.deb /tmp/ diff --git a/msa/web-ui/docker/Dockerfile b/msa/web-ui/docker/Dockerfile index 96a6861701..f99509893f 100644 --- a/msa/web-ui/docker/Dockerfile +++ b/msa/web-ui/docker/Dockerfile @@ -14,16 +14,13 @@ # limitations under the License. # -FROM node:16.13.1-bullseye-slim - -COPY start-web-ui.sh /tmp/ - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-web-ui.sh /usr/bin +FROM node:16.15.1-bullseye-slim ENV NODE_ENV production ENV DOCKER_MODE true +COPY start-web-ui.sh /tmp/ + WORKDIR ${pkg.installFolder} COPY ["src/package.json", "src/yarn.lock", "./"] @@ -33,9 +30,10 @@ COPY package/linux/conf ./config COPY web ./web COPY src/server.js ./ -RUN chown -R node:node ${pkg.installFolder} - -RUN yarn install --production && yarn cache clean --all +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-web-ui.sh /usr/bin \ + && chown -R node:node ${pkg.installFolder} \ + && yarn install --production && yarn cache clean --all USER node diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index c949555d46..2319caa092 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -80,7 +80,7 @@ install-node-and-yarn - v16.13.1 + v16.15.1 v1.22.17 diff --git a/ui-ngx/pom.xml b/ui-ngx/pom.xml index 66bf79951f..fce2719dc9 100644 --- a/ui-ngx/pom.xml +++ b/ui-ngx/pom.xml @@ -56,7 +56,7 @@ install-node-and-yarn - v16.13.0 + v16.15.1 v1.22.17 From aadfd1eb886c619c7a49062a75e5fee5349c2232 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 16 Jun 2022 13:47:02 +0300 Subject: [PATCH 244/262] Dockerfile improvements --- msa/tb/docker-cassandra/Dockerfile | 2 +- msa/tb/docker-postgres/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index f3fa3146c0..9d16d46ef5 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update \ && mv /tmp/stop-db.sh /usr/bin \ && dpkg -i /tmp/${pkg.name}.deb \ && rm /tmp/${pkg.name}.deb \ - && systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : \ + && (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) \ && mv /tmp/logback.xml ${pkg.installFolder}/conf \ && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf \ # postgres config diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index 24c9200ea4..5dfdba41bd 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -50,7 +50,7 @@ RUN apt-get update \ && mv /tmp/stop-db.sh /usr/bin \ && dpkg -i /tmp/${pkg.name}.deb \ && rm /tmp/${pkg.name}.deb \ - && systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : \ + && (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) \ && mv /tmp/logback.xml ${pkg.installFolder}/conf \ && mv /tmp/${pkg.name}.conf ${pkg.installFolder}/conf \ && mkdir -p $PGLOG \ From ad20b5d866c26abfa16beb8e9d9b2c0feadca090 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 15:00:43 +0300 Subject: [PATCH 245/262] Fix Ota Package Ids --- .../ie/importing/impl/AssetImportService.java | 2 +- .../impl/BaseEntityImportService.java | 9 +++++++-- .../importing/impl/CustomerImportService.java | 3 ++- .../importing/impl/DashboardImportService.java | 3 ++- .../ie/importing/impl/DeviceImportService.java | 7 ++++--- .../impl/DeviceProfileImportService.java | 9 ++++++--- .../impl/EntityViewImportService.java | 3 ++- .../importing/impl/RuleChainImportService.java | 18 ++++++++++++++++-- .../impl/WidgetsBundleImportService.java | 3 ++- application/src/main/resources/logback.xml | 3 +++ 10 files changed, 45 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index dd34160c6c..2a0dd99ab2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -44,7 +44,7 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Asset prepareAndSave(EntitiesImportCtx ctx, Asset asset, Asset old, EntityExportData exportData, IdProvider idProvider) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 0567614c6b..dafe2bc9ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -64,6 +64,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; @Slf4j @@ -104,7 +105,7 @@ public abstract class BaseEntityImportService importResult, D exportData, IdProvider idProvider) throws ThingsboardException { @@ -346,4 +347,8 @@ public abstract class BaseEntityImportService T getOldEntityField(O oldEntity, Function getter){ + return oldEntity == null ? null : getter.apply(oldEntity); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index f74b187d1e..2d64f1f837 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -45,7 +46,7 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer prepareAndSave(EntitiesImportCtx ctx, Customer customer, Customer old, EntityExportData exportData, IdProvider idProvider) { if (!customer.isPublic()) { return customerService.saveCustomer(customer); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 5c7e686c4a..5129fd9248 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -71,7 +72,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Dashboard prepareAndSave(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData exportData, IdProvider idProvider) { var tenantId = ctx.getTenantId(); JsonNode configuration = dashboard.getConfiguration(); JsonNode entityAliases = configuration.get("entityAliases"); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 0ad482e925..e0d160a713 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -43,10 +44,10 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); - deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); - deviceProfile.setSoftwareId(idProvider.getInternalId(deviceProfile.getSoftwareId())); + deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId)); + deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId)); return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java index 4bd903c110..5a4a9b784b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -51,7 +52,7 @@ public class EntityViewImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected EntityView prepareAndSave(EntitiesImportCtx ctx, EntityView entityView, EntityView old, EntityExportData exportData, IdProvider idProvider) { entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); return entityViewService.saveEntityView(entityView); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index d32f9589f6..b9706610e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -17,9 +17,12 @@ package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.RuleChainId; @@ -41,6 +44,7 @@ import java.util.LinkedHashSet; import java.util.Optional; import java.util.UUID; +@Slf4j @Service @TbCoreComponent @RequiredArgsConstructor @@ -65,7 +69,8 @@ public class RuleChainImportService extends BaseEntityImportService { @@ -87,10 +92,19 @@ public class RuleChainImportService extends BaseEntityImportService + + + From 5c72f596a41e4423d977004a8cb0f3058beddee1 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 15:04:59 +0300 Subject: [PATCH 246/262] Remove test logging --- .../ie/importing/impl/RuleChainImportService.java | 12 +----------- application/src/main/resources/logback.xml | 6 +++--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index b9706610e0..e3089f7705 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -70,7 +70,6 @@ public class RuleChainImportService extends BaseEntityImportService { @@ -92,19 +91,10 @@ public class RuleChainImportService extends BaseEntityImportService - - - + + + From dc8798e0c23c83485826c3ef966f74796e319220 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 16 Jun 2022 15:21:45 +0300 Subject: [PATCH 247/262] Update vc-executor docker build script --- msa/vc-executor-docker/docker/Dockerfile | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile index f4ff25e0e5..af259d8f67 100644 --- a/msa/vc-executor-docker/docker/Dockerfile +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -14,22 +14,18 @@ # limitations under the License. # -FROM thingsboard/openjdk11 +FROM thingsboard/openjdk11:bullseye-slim COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ -RUN mkdir -p /home/thingsboard/.config/jgit -RUN chown -R ${pkg.user}:${pkg.user} /home/thingsboard - -RUN chmod a+x /tmp/*.sh \ - && mv /tmp/start-tb-vc-executor.sh /usr/bin - -RUN yes | dpkg -i /tmp/${pkg.name}.deb -RUN rm /tmp/${pkg.name}.deb - -RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : - -RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar +RUN mkdir -p /home/thingsboard/.config/jgit \ + && chown -R ${pkg.user}:${pkg.user} /home/thingsboard \ + && chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar USER ${pkg.user} From 493b8f2f7b0a132162b06c573b018a7ab7385b5c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 17:53:05 +0300 Subject: [PATCH 248/262] Improved comparison of the objects before import --- .../main/data/upgrade/3.3.4/schema_update.sql | 1 + .../DefaultEntitiesExportImportService.java | 3 - .../sync/ie/EntitiesExportImportService.java | 3 - .../DefaultExportableEntitiesService.java | 4 - .../ie/exporting/EntityExportService.java | 2 - .../exporting/ExportableEntitiesService.java | 3 - .../ie/exporting/impl/AssetExportService.java | 3 - .../impl/BaseEntityExportService.java | 4 - .../impl/DashboardExportService.java | 2 - .../impl/DefaultEntityExportService.java | 4 - .../exporting/impl/DeviceExportService.java | 4 +- .../impl/DeviceProfileExportService.java | 2 - .../impl/EntityViewExportService.java | 2 - .../impl/RuleChainExportService.java | 4 +- .../impl/WidgetsBundleExportService.java | 2 - .../ie/importing/EntityImportService.java | 4 +- .../ie/importing/impl/AssetImportService.java | 23 +++++- .../impl/BaseEntityImportService.java | 82 ++++++++++++++----- .../importing/impl/CustomerImportService.java | 26 ++++-- .../impl/DashboardImportService.java | 15 +++- .../importing/impl/DeviceImportService.java | 42 +++++++++- .../impl/DeviceProfileImportService.java | 18 ++-- .../impl/EntityViewImportService.java | 21 ++++- .../impl/RuleChainImportService.java | 20 ++++- .../impl/WidgetsBundleImportService.java | 14 +++- .../DefaultEntitiesVersionControlService.java | 15 +++- .../data/sync/ie/EntityImportResult.java | 4 + .../resources/sql/schema-entities-idx.sql | 2 + 28 files changed, 232 insertions(+), 97 deletions(-) diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index 26df9f84ff..d449b418f3 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -38,6 +38,7 @@ CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, e CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 807831ecff..b2abe2a581 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -28,15 +28,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.apiusage.RateLimitService; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 349509b09e..8657528dac 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -20,10 +20,7 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 822ce9025c..6e06144ae2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -22,7 +22,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -45,10 +44,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.Collection; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index ac08425c10..505d1b2a3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -18,8 +18,6 @@ package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index c247c51efa..0a99979e8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -17,14 +17,11 @@ package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; public interface ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java index 3c60fd2976..8875491d42 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -15,14 +15,11 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 26c18cf37d..f233672297 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -19,13 +19,9 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; -import java.util.Optional; import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 1064962082..99a6a36494 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -20,9 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 4f748ba9d8..0ba160adbc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -28,19 +28,15 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 244f691349..bc5136bee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -20,11 +20,9 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java index 3e076ddc13..46423c66ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java @@ -19,9 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java index 1f3459f18f..3fff890d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java @@ -19,9 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index 9af29cceba..464e5fbf3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -19,12 +19,10 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java index 3065aae451..d3dd23910f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -18,9 +18,7 @@ package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index 4e7d4d8495..9437f21e60 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -19,10 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 2a0dd99ab2..cd9dc17d60 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -23,11 +23,10 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @@ -44,7 +43,12 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Asset prepare(EntitiesImportCtx ctx, Asset asset, Asset old, EntityExportData exportData, IdProvider idProvider) { + return asset; + } + + @Override + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { return assetService.saveAsset(asset); } @@ -56,6 +60,19 @@ public class AssetImportService extends BaseEntityImportService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { // TbStopWatch sw = TbStopWatch.create("find"); EntityImportResult importResult = new EntityImportResult<>(); + importResult.setEntityType(getEntityType()); IdProvider idProvider = new IdProvider(ctx, importResult); E entity = exportData.getEntity(); - E existingEntity = findExistingEntity(ctx, entity, idProvider); - entity.setExternalId(entity.getId()); + E existingEntity = findExistingEntity(ctx, entity, idProvider); + importResult.setOldEntity(existingEntity); + setOwner(ctx.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); @@ -104,17 +105,27 @@ public abstract class BaseEntityImportService importResult, D exportData, IdProvider idProvider) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); - importResult.addSendEventsCallback(() -> { - onEntitySaved(ctx.getUser(), savedEntity, oldEntity); - }); + if (importResult.isCreated() || importResult.isUpdated()) { + importResult.addSendEventsCallback(() -> onEntitySaved(ctx.getUser(), savedEntity, oldEntity)); + } if (ctx.isUpdateRelations() && exportData.getRelations() != null) { importRelations(ctx, exportData.getRelations(), importResult, idProvider); } if (ctx.isSaveAttributes() && exportData.getAttributes() != null) { + if (exportData.getAttributes().values().stream().anyMatch(d -> !d.isEmpty())) { + importResult.setUpdatedRelatedEntities(true); + } importAttributes(ctx.getUser(), exportData.getAttributes(), importResult); } } @@ -173,6 +212,7 @@ public abstract class BaseEntityImportService { entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, @@ -185,8 +225,10 @@ public abstract class BaseEntityImportService T getOldEntityField(O oldEntity, Function getter){ + protected T getOldEntityField(O oldEntity, Function getter) { return oldEntity == null ? null : getter.apply(oldEntity); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index 2d64f1f837..1dc476340f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -19,17 +19,15 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @@ -46,16 +44,30 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer prepare(EntitiesImportCtx ctx, Customer customer, Customer old, EntityExportData exportData, IdProvider idProvider) { + if (customer.isPublic()) { + Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId()); + publicCustomer.setExternalId(customer.getExternalId()); + return publicCustomer; + } else { + return customer; + } + } + + @Override + protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData exportData, IdProvider idProvider) { if (!customer.isPublic()) { return customerService.saveCustomer(customer); } else { - Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId()); - publicCustomer.setExternalId(customer.getExternalId()); - return customerDao.save(ctx.getTenantId(), publicCustomer); + return customerDao.save(ctx.getTenantId(), customer); } } + @Override + protected Customer deepCopy(Customer customer) { + return new Customer(customer); + } + @Override protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException { super.onEntitySaved(user, savedCustomer, oldCustomer); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 5129fd9248..7a4ea5a1ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -72,8 +71,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - var tenantId = ctx.getTenantId(); + protected Dashboard prepare(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData exportData, IdProvider idProvider) { JsonNode configuration = dashboard.getConfiguration(); JsonNode entityAliases = configuration.get("entityAliases"); if (entityAliases != null && entityAliases.isObject()) { @@ -90,6 +88,12 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) @@ -124,6 +128,11 @@ public class DashboardImportService extends BaseEntityImportService { private final DeviceService deviceService; + private final DeviceCredentialsService credentialsService; @Override protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) { @@ -44,10 +44,28 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId)); deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId)); + return deviceProfile; + } + + @Override + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { return deviceProfileService.saveDeviceProfile(deviceProfile); } @@ -72,6 +73,11 @@ public class DeviceProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected EntityView prepare(EntitiesImportCtx ctx, EntityView entityView, EntityView old, EntityExportData exportData, IdProvider idProvider) { entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); + return entityView; + } + + @Override + protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData exportData, IdProvider idProvider) { return entityViewService.saveEntityView(entityView); } @@ -66,6 +70,19 @@ public class EntityViewImportService extends BaseEntityImportService { @@ -90,13 +88,24 @@ public class RuleChainImportService extends BaseEntityImportService(ctx.getResults().values())); @@ -341,7 +341,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont continue; } - ctx.registerResult(entityType, importResult.getOldEntity() == null); + registerResult(ctx, entityType, importResult); ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } @@ -359,7 +359,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont ctx.setSettings(settings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); - ctx.registerResult(externalId.getEntityType(), importResult.getOldEntity() == null); + registerResult(ctx, externalId.getEntityType(), importResult); ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } catch (Exception e) { @@ -540,4 +540,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return message; } + private void registerResult(EntitiesImportCtx ctx, EntityType entityType, EntityImportResult importResult) { + if (importResult.isCreated()) { + ctx.registerResult(entityType, true); + } else if (importResult.isUpdated() || importResult.isUpdatedRelatedEntities()) { + ctx.registerResult(entityType, false); + } + } + + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 4a3f8fc57e..239777f827 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -33,6 +33,10 @@ public class EntityImportResult> private Boolean updatedAllExternalIds; + private boolean created; + private boolean updated; + private boolean updatedRelatedEntities; + public void addSaveReferencesCallback(ThrowingRunnable callback) { this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); } diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index b47462f94b..d1387a23ad 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -58,6 +58,8 @@ CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tena CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); + CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); From 49a83d3e18808b867d1a74fbf203fabb1eb83561 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 16 Jun 2022 18:17:12 +0300 Subject: [PATCH 249/262] Fix tests --- .../sync/ie/BaseExportImportServiceTest.java | 1 + .../sync/ie/ExportImportServiceSqlTest.java | 19 ++++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java index 06f8bcd281..7fa019f64c 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -357,6 +357,7 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData, EntityImportSettings importSettings) throws Exception { EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings); + ctx.setFetchAllUUIDs(true); exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class); EntityImportResult importResult = exportImportService.importEntity(ctx, exportData); exportImportService.saveReferencesAndRelations(ctx); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 63029023f3..facae5a823 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -73,8 +72,6 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { @SpyBean private EntityActionService entityActionService; @SpyBean - private TbClusterService clusterService; - @SpyBean private OtaPackageStateService otaPackageStateService; @Test @@ -449,7 +446,7 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { importEntity(tenantAdmin2, entitiesExportData.get(EntityType.CUSTOMER)); verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), any(), eq(ActionType.UPDATED), isNull()); - verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); Asset importedAsset = (Asset) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), @@ -457,12 +454,12 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)); verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), any(), eq(ActionType.UPDATED), isNull()); - verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.RULE_CHAIN)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), any(), eq(ActionType.ADDED), isNull()); - verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DASHBOARD)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), @@ -471,17 +468,17 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE_PROFILE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), any(), eq(ActionType.ADDED), isNull()); - verify(clusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); - verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); - verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(tbClusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); Device importedDevice = (Device) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), any(), eq(ActionType.ADDED), isNull()); - verify(clusterService).onDeviceUpdated(eq(importedDevice), isNull()); + verify(tbClusterService).onDeviceUpdated(eq(importedDevice), isNull()); importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)); - verify(clusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + verify(tbClusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); } } From 86c5d18116d6db4ac82bb6c1a784975f96073850 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 17 Jun 2022 11:37:28 +0300 Subject: [PATCH 250/262] Tests for external ids in export data --- .../sync/ie/BaseExportImportServiceTest.java | 77 +++++++++++++++++++ .../sync/ie/ExportImportServiceSqlTest.java | 62 +++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java index 7fa019f64c..c4de720b9c 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -15,11 +15,14 @@ */ package org.thingsboard.server.service.sync.ie; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import org.junit.After; import org.junit.Before; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -27,6 +30,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.OtaPackage; @@ -38,6 +42,7 @@ import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -64,6 +69,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -101,6 +107,8 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest protected RelationService relationService; @Autowired protected TenantService tenantService; + @Autowired + protected EntityViewService entityViewService; protected TenantId tenantId1; protected User tenantAdmin1; @@ -248,6 +256,29 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest return dashboard; } + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name, AssetId assetForEntityAlias) { + Dashboard dashboard = createDashboard(tenantId, customerId, name); + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + assetForEntityAlias.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + return dashboardService.saveDashboard(dashboard); + } + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); @@ -257,6 +288,42 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); } } + protected RuleChain createRuleChain(TenantId tenantId, String name, EntityId originatorId) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Generator 1"); + ruleNode1.setType(TbMsgGeneratorNode.class.getName()); + ruleNode1.setDebugMode(true); + TbMsgGeneratorNodeConfiguration configuration1 = new TbMsgGeneratorNodeConfiguration(); + configuration1.setOriginatorType(originatorId.getEntityType()); + configuration1.setOriginatorId(originatorId.getId().toString()); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } protected RuleChain createRuleChain(TenantId tenantId, String name) { RuleChain ruleChain = new RuleChain(); @@ -313,6 +380,16 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest } } + protected EntityView createEntityView(TenantId tenantId, CustomerId customerId, EntityId entityId, String name) { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setEntityId(entityId); + entityView.setCustomerId(customerId); + entityView.setName(name); + entityView.setType("A"); + return entityViewService.saveEntityView(entityView); + } + protected EntityRelation createRelation(EntityId from, EntityId to) { EntityRelation relation = new EntityRelation(); relation.setFrom(from); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index facae5a823..7503e5574c 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -23,16 +23,26 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -40,17 +50,21 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.ota.OtaPackageStateService; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,6 +76,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.verify; @DaoSqlTest @@ -481,4 +496,51 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { verify(tbClusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); } + @Test + public void testExternalIdsInExportData() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, customer.getId(), "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1", asset.getId()); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1", asset.getId()); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); + EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + + Map ids = new HashMap<>(); + for (EntityId entityId : List.of(customer.getId(), asset.getId(), ruleChain.getId(), dashboard.getId(), + deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { + EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); + EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() + .saveCredentials(false) + .build()); + ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); + } + + Asset exportedAsset = (Asset) exportEntity(tenantAdmin2, (AssetId) ids.get(asset.getId())).getEntity(); + assertThat(exportedAsset.getCustomerId()).isEqualTo(customer.getId()); + + EntityExportData ruleChainExportData = exportEntity(tenantAdmin2, (RuleChainId) ids.get(ruleChain.getId())); + TbMsgGeneratorNodeConfiguration exportedRuleNodeConfig = ((RuleChainExportData) ruleChainExportData).getMetaData().getNodes().stream() + .filter(node -> node.getType().equals(TbMsgGeneratorNode.class.getName())).findFirst() + .map(RuleNode::getConfiguration).map(config -> JacksonUtil.treeToValue(config, TbMsgGeneratorNodeConfiguration.class)).orElse(null); + assertThat(exportedRuleNodeConfig.getOriginatorId()).isEqualTo(asset.getId().toString()); + + Dashboard exportedDashboard = (Dashboard) exportEntity(tenantAdmin2, (DashboardId) ids.get(dashboard.getId())).getEntity(); + assertThat(exportedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(shortCustomerInfo -> { + assertThat(shortCustomerInfo.getCustomerId()).isEqualTo(customer.getId()); + }); + + DeviceProfile exportedDeviceProfile = (DeviceProfile) exportEntity(tenantAdmin2, (DeviceProfileId) ids.get(deviceProfile.getId())).getEntity(); + assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); + assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); + + Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); + assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); + } + } From 2dad98b2f7a7fadcebb9742dddf96281b12e8aa5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 17 Jun 2022 11:54:20 +0300 Subject: [PATCH 251/262] Improvement to the Rule Chain diff --- .../impl/BaseEntityImportService.java | 18 +++++------------- .../importing/impl/RuleChainImportService.java | 13 +++++++++---- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 01a341bb4c..783772719c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -86,7 +86,6 @@ public abstract class BaseEntityImportService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { -// TbStopWatch sw = TbStopWatch.create("find"); EntityImportResult importResult = new EntityImportResult<>(); importResult.setEntityType(getEntityType()); IdProvider idProvider = new IdProvider(ctx, importResult); @@ -107,10 +106,9 @@ public abstract class BaseEntityImportService { ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); }); + //TODO: lookup rule node id based on external rule node id. ruleChain.setFirstRuleNodeId(null); return ruleChain; } @@ -100,10 +101,14 @@ public class RuleChainImportService extends BaseEntityImportService Date: Fri, 17 Jun 2022 12:06:51 +0300 Subject: [PATCH 252/262] Fix testExternalIdsInExportData --- .../service/sync/ie/ExportImportServiceSqlTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 7503e5574c..295879c7b0 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -529,6 +529,9 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { assertThat(exportedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(shortCustomerInfo -> { assertThat(shortCustomerInfo.getCustomerId()).isEqualTo(customer.getId()); }); + String exportedEntityAliasAssetId = exportedDashboard.getConfiguration().get("entityAliases").elements().next() + .get("filter").get("entityList").elements().next().asText(); + assertThat(exportedEntityAliasAssetId).isEqualTo(asset.getId().toString()); DeviceProfile exportedDeviceProfile = (DeviceProfile) exportEntity(tenantAdmin2, (DeviceProfileId) ids.get(deviceProfile.getId())).getEntity(); assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); @@ -541,6 +544,12 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); + + deviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(deviceProfile); + DeviceProfile importedDeviceProfile = deviceProfileService.findDeviceProfileById(tenantId2, (DeviceProfileId) ids.get(deviceProfile.getId())); + importedDeviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(importedDeviceProfile); } } From f9bfe6eb6fa3bfc2990c4804670c0f846bc0c4de Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 17 Jun 2022 12:10:02 +0300 Subject: [PATCH 253/262] Fix 2fa error msg --- .../security/auth/mfa/provider/impl/SmsTwoFaProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java index 63d3233928..419a60b3b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java @@ -63,7 +63,7 @@ public class SmsTwoFaProvider extends OtpBasedTwoFaProvider Date: Fri, 17 Jun 2022 12:46:31 +0300 Subject: [PATCH 254/262] Fix getAlarms for entity for customer - filter by customer id. --- .../org/thingsboard/server/controller/AlarmController.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 939e9da8a0..b0ed1ed01d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -226,7 +226,11 @@ public class AlarmController extends BaseController { TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); try { - return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); + if (getCurrentUser().isCustomerUser()) { + return checkNotNull(alarmService.findCustomerAlarms(getCurrentUser().getTenantId(), getCurrentUser().getCustomerId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); + } else { + return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); + } } catch (Exception e) { throw handleException(e); } From 4036725d10f8676e3fbf5515678ac94c4aba720b Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 17 Jun 2022 13:55:03 +0300 Subject: [PATCH 255/262] Fix entities deletion order when deleting tenant --- .../org/thingsboard/server/dao/tenant/TenantServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 61c037e2d6..0590286714 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -149,9 +149,9 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + entityViewService.deleteEntityViewsByTenantId(tenantId); customerService.deleteCustomersByTenantId(tenantId); widgetsBundleService.deleteWidgetsBundlesByTenantId(tenantId); - entityViewService.deleteEntityViewsByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); deviceService.deleteDevicesByTenantId(tenantId); deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); From d33c783b66d34256b13bb1ddc27caac030adf904 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 17 Jun 2022 14:57:37 +0300 Subject: [PATCH 256/262] Fix circular bean dependency --- .../thingsboard/server/config/RateLimitProcessingFilter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 5cc9747ed9..1415ff4ece 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -17,6 +17,7 @@ package org.thingsboard.server.config; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -47,6 +48,7 @@ public class RateLimitProcessingFilter extends GenericFilterBean { private ThingsboardErrorResponseHandler errorResponseHandler; @Autowired + @Lazy private TbTenantProfileCache tenantProfileCache; private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); From fa5335130b47eb103f23b87a1c0e02e83d3565cc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 17 Jun 2022 16:10:29 +0300 Subject: [PATCH 257/262] Entities import/export for CE --- .../main/data/upgrade/3.3.4/schema_update.sql | 6 ++ .../impl/RuleChainExportService.java | 9 ++- .../impl/DashboardImportService.java | 3 +- .../impl/DeviceProfileImportService.java | 7 +++ .../impl/RuleChainImportService.java | 55 ++++++++++++++----- .../sync/vc/data/EntitiesExportCtx.java | 2 +- .../server/dao/relation/RelationService.java | 2 - .../server/common/data/BaseData.java | 4 +- .../server/common/data/DashboardInfo.java | 35 +++++------- .../server/common/data/rule/RuleNode.java | 3 + .../server/dao/model/sql/RuleNodeEntity.java | 9 +++ .../dao/relation/BaseRelationService.java | 40 +------------- .../server/dao/rule/BaseRuleChainService.java | 2 + .../server/dao/rule/RuleNodeDao.java | 3 + .../settings/AdminSettingsServiceImpl.java | 1 + .../dao/sql/alarm/EntityAlarmRepository.java | 7 ++- .../ComponentDescriptorRepository.java | 7 ++- .../dao/sql/relation/RelationRepository.java | 17 ++++-- .../server/dao/sql/rule/JpaRuleNodeDao.java | 7 +++ .../dao/sql/rule/RuleNodeRepository.java | 13 ++++- .../dao/sql/settings/JpaAdminSettingsDao.java | 1 + .../usagerecord/ApiUsageStateRepository.java | 3 +- .../sql/user/UserAuthSettingsRepository.java | 7 ++- .../resources/sql/schema-entities-idx.sql | 4 ++ .../main/resources/sql/schema-entities.sql | 3 +- .../dao/service/BaseRelationServiceTest.java | 2 +- 26 files changed, 154 insertions(+), 98 deletions(-) diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index d449b418f3..69df2afdc9 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -22,6 +22,8 @@ ALTER TABLE asset ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE rule_chain ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_node + ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE customer @@ -35,11 +37,14 @@ CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_ CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id); CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type); + ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; @@ -64,3 +69,4 @@ CREATE TABLE IF NOT EXISTS user_auth_settings ( user_id uuid UNIQUE NOT NULL CONSTRAINT fk_user_auth_settings_user_id REFERENCES tb_user(id), two_fa_settings varchar ); + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index c3f8071191..a28f8b8847 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -22,8 +22,8 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; @@ -47,6 +47,10 @@ public class RuleChainExportService extends BaseEntityExportService { ruleNode.setRuleChainId(null); + ctx.putExternalId(ruleNode.getId(), ruleNode.getExternalId()); + ruleNode.setId(ctx.getExternalId(ruleNode.getId())); + ruleNode.setCreatedTime(0); + ruleNode.setExternalId(null); JsonNode ruleNodeConfig = ruleNode.getConfiguration(); String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); @@ -59,6 +63,9 @@ public class RuleChainExportService extends BaseEntityExportService HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); private final RuleChainService ruleChainService; + private final RuleNodeDao ruleNodeDao; @Override protected void setOwner(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { @@ -69,26 +75,44 @@ public class RuleChainImportService extends BaseEntityImportService { - ruleNode.setId(null); - ruleNode.setRuleChainId(null); + List ruleNodes = Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()); + if (old != null) { +// boolean original = old.getId().equals(old.getExternalId()); + List nodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + List existing = ruleNodeDao.findByExternalIds(old.getId(), nodeIds); + existing.forEach(node -> ctx.putInternalId(node.getExternalId(), node.getId())); + ruleNodes.forEach(node -> { + node.setRuleChainId(old.getId()); +// if (!original) { + node.setExternalId(node.getId()); +// } + node.setId((RuleNodeId) ctx.getInternalId(node.getId())); + }); + } else { + ruleNodes.forEach(node -> { + node.setRuleChainId(null); + node.setExternalId(node.getId()); + node.setId(null); + }); + } - JsonNode ruleNodeConfig = ruleNode.getConfiguration(); - String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) - .map(entityId -> entityId.getId().toString()) - .orElse(uuid); - }); - ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); - ruleNode.setConfiguration(ruleNodeConfig); - }); + ruleNodes.forEach(ruleNode -> { + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); + String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) + .map(entityId -> entityId.getId().toString()) + .orElse(uuid); + }); + ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); + ruleNode.setConfiguration(ruleNodeConfig); + }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); }); - //TODO: lookup rule node id based on external rule node id. - ruleChain.setFirstRuleNodeId(null); + if (ruleChain.getFirstRuleNodeId() != null) { + ruleChain.setFirstRuleNodeId((RuleNodeId) ctx.getInternalId(ruleChain.getFirstRuleNodeId())); + } return ruleChain; } @@ -106,6 +130,7 @@ public class RuleChainImportService extends BaseEntityImportService { public void putExternalId(EntityId internalId, EntityId externalId) { log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); - externalIdMap.put(internalId, externalId); + externalIdMap.put(internalId, externalId != null ? externalId : internalId); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index bc9980fcd8..ee04e6046b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -55,8 +55,6 @@ public interface RelationService { void deleteEntityRelations(TenantId tenantId, EntityId entity); - ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entity); - List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); ListenableFuture> findByFromAsync(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java index c8cf730ead..f2ada352e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java @@ -65,9 +65,7 @@ public abstract class BaseData extends IdBased implement if (getClass() != obj.getClass()) return false; BaseData other = (BaseData) obj; - if (createdTime != other.createdTime) - return false; - return true; + return createdTime == other.createdTime; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java index e603f68c77..0a6d2260d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; @@ -26,6 +27,7 @@ import org.thingsboard.server.common.data.validation.NoXss; import javax.validation.Valid; import java.util.HashSet; +import java.util.Objects; import java.util.Set; @ApiModel @@ -63,7 +65,7 @@ public class DashboardInfo extends SearchTextBased implements HasNa @ApiModelProperty(position = 1, value = "JSON object with the dashboard Id. " + "Specify existing dashboard Id to update the dashboard. " + "Referencing non-existing dashboard id will cause error. " + - "Omit this field to create new dashboard." ) + "Omit this field to create new dashboard.") @Override public DashboardId getId() { return super.getId(); @@ -133,7 +135,6 @@ public class DashboardInfo extends SearchTextBased implements HasNa return this.assignedCustomers != null && this.assignedCustomers.contains(new ShortCustomerInfo(customerId, null, false)); } - public ShortCustomerInfo getAssignedCustomerInfo(CustomerId customerId) { if (this.assignedCustomers != null) { for (ShortCustomerInfo customerInfo : this.assignedCustomers) { @@ -201,25 +202,17 @@ public class DashboardInfo extends SearchTextBased implements HasNa } @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - DashboardInfo other = (DashboardInfo) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + DashboardInfo that = (DashboardInfo) o; + return mobileHide == that.mobileHide + && Objects.equals(tenantId, that.tenantId) + && Objects.equals(title, that.title) + && Objects.equals(image, that.image) + && Objects.equals(assignedCustomers, that.assignedCustomers) + && Objects.equals(mobileOrder, that.mobileOrder); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java index fa2760dbc3..d9dce57f46 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java @@ -53,6 +53,8 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl @JsonIgnore private byte[] configurationBytes; + private RuleNodeId externalId; + public RuleNode() { super(); } @@ -68,6 +70,7 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl this.name = ruleNode.getName(); this.debugMode = ruleNode.isDebugMode(); this.setConfiguration(ruleNode.getConfiguration()); + this.externalId = ruleNode.getExternalId(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index 04b7fd4ecb..70e8497c31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -64,6 +64,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Column(name = ModelConstants.DEBUG_MODE) private boolean debugMode; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleNodeEntity() { } @@ -81,6 +84,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex this.searchText = ruleNode.getName(); this.configuration = ruleNode.getConfiguration(); this.additionalInfo = ruleNode.getAdditionalInfo(); + if (ruleNode.getExternalId() != null) { + this.externalId = ruleNode.getExternalId().getId(); + } } @Override @@ -105,6 +111,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex ruleNode.setDebugMode(debugMode); ruleNode.setConfiguration(configuration); ruleNode.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleNode.setExternalId(new RuleNodeId(externalId)); + } return ruleNode; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 7698adfda2..cef42071df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -25,6 +25,7 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.thingsboard.server.cache.TbTransactionalCache; @@ -180,6 +181,7 @@ public class BaseRelationService implements RelationService { return future; } + @Transactional @Override public void deleteEntityRelations(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteEntityRelations [{}]", entityId); @@ -208,44 +210,6 @@ public class BaseRelationService implements RelationService { } } - @Override - public ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entityId) { - log.trace("Executing deleteEntityRelationsAsync [{}]", entityId); - validate(entityId); - List>> inboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelationsList.add(executor.submit(() -> relationDao.findAllByTo(tenantId, entityId, typeGroup))); - } - - ListenableFuture>> inboundRelations = Futures.allAsList(inboundRelationsList); - - List>> outboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelationsList.add(executor.submit(() -> relationDao.findAllByFrom(tenantId, entityId, typeGroup))); - } - - ListenableFuture>> outboundRelations = Futures.allAsList(outboundRelationsList); - - ListenableFuture> inboundDeletions = Futures.transformAsync(inboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, true); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, false); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions); - - return Futures.transform(Futures.transformAsync(deletionsFuture, - (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId), - MoreExecutors.directExecutor()), - result -> null, MoreExecutors.directExecutor()); - } - private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, boolean deleteFromDb) { List> results = new ArrayList<>(); for (List relationList : relations) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 7660e489fc..1a3a0d02ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -301,6 +301,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } } } + Collections.sort(ruleChainMetaData.getConnections(), Comparator.comparingInt(NodeConnectionInfo::getFromIndex) + .thenComparing(NodeConnectionInfo::getToIndex).thenComparing(NodeConnectionInfo::getType)); return ruleChainMetaData; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index bd3efcd205..4fb15e5721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -33,5 +34,7 @@ public interface RuleNodeDao extends Dao { PageData findAllRuleNodesByType(String type, PageLink pageLink); + List findByExternalIds(RuleChainId ruleChainId, List externalIds); + void deleteByIdIn(List ruleNodeIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 7d6f931ef1..28b4c97876 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index 43d43a5f8e..0b409d9256 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; @@ -28,5 +31,7 @@ public interface EntityAlarmRepository extends JpaRepository findAllByAlarmId(UUID alarmId); @Transactional - void deleteByEntityId(UUID id); + @Modifying + @Query("DELETE FROM EntityAlarmEntity e where e.entityId = :entityId") + void deleteByEntityId(@Param("entityId") UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java index 1a549436d0..4ec4a562c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; @@ -46,5 +48,8 @@ public interface ComponentDescriptorRepository extends JpaRepository findAllByFromIdAndFromTypeAndRelationTypeGroupIn(UUID fromId, - String fromType, - List relationTypeGroups); + String fromType, + List relationTypeGroups); List findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup(UUID fromId, String fromType, @@ -49,8 +50,8 @@ public interface RelationRepository String relationTypeGroup); List findAllByToIdAndToTypeAndRelationTypeGroupIn(UUID toId, - String toType, - List relationTypeGroups); + String toType, + List relationTypeGroups); List findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup(UUID toId, String toType, @@ -72,9 +73,13 @@ public interface RelationRepository void deleteById(RelationCompositeKey id); @Transactional - void deleteByFromIdAndFromType(UUID fromId, String fromType); + @Modifying + @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType") + void deleteByFromIdAndFromType(@Param("fromId") UUID fromId, @Param("fromType") String fromType); @Transactional - void deleteByToIdAndToTypeAndRelationTypeGroupIn(UUID fromId, String fromType, List relationTypeGroups); + @Modifying + @Query("DELETE FROM RelationEntity r where r.toId = :toId and r.toType = :toType and r.relationTypeGroup in :relationTypeGroups") + void deleteByToIdAndToTypeAndRelationTypeGroupIn(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroups") List relationTypeGroups); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 47020657e6..7fc1229cd3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -66,6 +67,12 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao findByExternalIds(RuleChainId ruleChainId, List externalIds) { + return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByRuleChainIdAndExternalIdIn(ruleChainId.getId(), + externalIds.stream().map(RuleNodeId::getId).collect(Collectors.toList()))); + } + @Override public void deleteByIdIn(List ruleNodeIds) { ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index 9fee64824b..e6a08b910a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.rule; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import java.util.List; @@ -31,14 +33,19 @@ public interface RuleNodeRepository extends JpaRepository "(select id from RuleChainEntity rc WHERE rc.tenantId = :tenantId) " + "AND r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") List findRuleNodesByTenantIdAndType(@Param("tenantId") UUID tenantId, - @Param("ruleType") String ruleType, - @Param("searchText") String searchText); + @Param("ruleType") String ruleType, + @Param("searchText") String searchText); @Query("SELECT r FROM RuleNodeEntity r WHERE r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") Page findAllRuleNodesByType(@Param("ruleType") String ruleType, @Param("searchText") String searchText, Pageable pageable); - void deleteByIdIn(List ids); + List findRuleNodesByRuleChainIdAndExternalIdIn(UUID ruleChainId, List externalIds); + + @Transactional + @Modifying + @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") + void deleteByIdIn(@Param("ids") List ids); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 719dc420a9..bd815bc410 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -62,6 +62,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDao Date: Sat, 18 Jun 2022 09:28:54 +0300 Subject: [PATCH 258/262] Fix re-import of rule chains, and different not related test --- .../customer/DefaultTbCustomerService.java | 2 +- .../install/update/RateLimitsUpdater.java | 2 +- .../impl/BaseEntityImportService.java | 1 + .../impl/DashboardImportService.java | 2 +- .../impl/RuleChainImportService.java | 15 ++--- .../DefaultEntitiesVersionControlService.java | 8 ++- .../sync/vc/data/EntitiesImportCtx.java | 6 +- .../BaseCustomerControllerTest.java | 3 +- .../sync/ie/BaseExportImportServiceTest.java | 2 +- .../sync/ie/ExportImportServiceSqlTest.java | 56 ++++++++++++++----- .../data/sync/ie/EntityImportResult.java | 2 +- .../server/dao/rule/BaseRuleChainService.java | 6 +- .../profile/TbDeviceProfileNodeTest.java | 4 +- 13 files changed, 76 insertions(+), 33 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java index 306bb35d43..536e1be976 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java @@ -43,7 +43,7 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements try { Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); vcService.autoCommit(user, savedCustomer.getId()); - notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, savedCustomer.getId(), actionType, user); + notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user); return savedCustomer; } catch (Exception e) { notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.CUSTOMER), customer, null, actionType, user, e); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index 054f68f71d..c136f97b52 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 783772719c..31583e0ee9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -87,6 +87,7 @@ public abstract class BaseEntityImportService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { EntityImportResult importResult = new EntityImportResult<>(); + ctx.setCurrentImportResult(importResult); importResult.setEntityType(getEntityType()); IdProvider idProvider = new IdProvider(ctx, importResult); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 636114b13e..4d767a5f61 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -81,7 +81,7 @@ public class DashboardImportService extends BaseEntityImportService { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFinalImportAttempt(), HINTS) .map(entityId -> entityId.getId().toString()).orElse(uuid); })); ((ObjectNode) entityAlias).set(field, newFieldValue); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index cfcb4c0498..a9e230a531 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -77,15 +77,12 @@ public class RuleChainImportService extends BaseEntityImportService ruleNodes = Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()); if (old != null) { -// boolean original = old.getId().equals(old.getExternalId()); List nodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); List existing = ruleNodeDao.findByExternalIds(old.getId(), nodeIds); existing.forEach(node -> ctx.putInternalId(node.getExternalId(), node.getId())); ruleNodes.forEach(node -> { node.setRuleChainId(old.getId()); -// if (!original) { node.setExternalId(node.getId()); -// } node.setId((RuleNodeId) ctx.getInternalId(node.getId())); }); } else { @@ -99,7 +96,7 @@ public class RuleChainImportService extends BaseEntityImportService { JsonNode ruleNodeConfig = ruleNode.getConfiguration(); String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFinalImportAttempt(), HINTS) .map(entityId -> entityId.getId().toString()) .orElse(uuid); }); @@ -119,9 +116,13 @@ public class RuleChainImportService extends BaseEntityImportService importResult = exportImportService.importEntity(ctx, entityData); exportImportService.saveReferencesAndRelations(ctx); @@ -329,6 +330,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException(e); } for (EntityExportData entityData : entityDataList) { + EntityExportData reimportBackup = JacksonUtil.clone(entityData); log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { @@ -336,8 +338,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new LoadEntityException(entityData, e); } - if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(entityData, ctx.getSettings())); + if (!importResult.isUpdatedAllExternalIds()) { + ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(reimportBackup, ctx.getSettings())); continue; } @@ -351,7 +353,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"rawtypes", "unchecked"}) private void reimport(EntitiesImportCtx ctx) { - ctx.setFetchAllUUIDs(true); + ctx.setFinalImportAttempt(true); ctx.getToReimport().forEach((externalId, task) -> { try { EntityExportData entityData = task.getData(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 74d09a5c3b..c781f3cd18 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.service.security.model.SecurityUser; @@ -52,8 +53,9 @@ public class EntitiesImportCtx { private final Set relations = new LinkedHashSet<>(); - private boolean fetchAllUUIDs = false; + private boolean finalImportAttempt = false; private EntityImportSettings settings; + private EntityImportResult currentImportResult; public EntitiesImportCtx(SecurityUser user, String versionId) { this(user, versionId, null); @@ -134,4 +136,6 @@ public class EntitiesImportCtx { return notFoundIds.contains(externalId); } + + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index 1f94fa4784..45c8d72529 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -108,7 +109,7 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doPost("/api/customer", savedCustomer, Customer.class); testNotifyEntityAllOneTime(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(), - savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + new CustomerId(CustomerId.NULL_UUID), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java index c4de720b9c..9b34eaae25 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -434,7 +434,7 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData, EntityImportSettings importSettings) throws Exception { EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings); - ctx.setFetchAllUUIDs(true); + ctx.setFinalImportAttempt(true); exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class); EntityImportResult importResult = exportImportService.importEntity(ctx, exportData); exportImportService.saveReferencesAndRelations(ctx); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 7503e5574c..d3b2f32657 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.Streams; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; @@ -271,7 +272,7 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { Dashboard importedDashboard = importEntity(tenantAdmin2, dashboardExportData).getSavedEntity(); Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() - .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) .map(JsonNode::asText).collect(Collectors.toSet()); assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); @@ -433,6 +434,12 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { } + @SuppressWarnings("rawTypes") + private static EntityExportData getAndClone(Map map, EntityType entityType) { + return JacksonUtil.clone(map.get(entityType)); + } + + @SuppressWarnings({"rawTypes", "unchecked"}) @Test public void testEntityEventsOnImport() throws Exception { Customer customer = createCustomer(tenantId1, "Customer 1"); @@ -443,7 +450,7 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), - ruleChain.getId(), dashboard.getId(), deviceProfile.getId()) + ruleChain.getId(), dashboard.getId(), deviceProfile.getId()) .map(entityId -> { try { return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() @@ -455,32 +462,50 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { }) .collect(Collectors.toMap(EntityExportData::getEntityType, d -> d)); - Customer importedCustomer = (Customer) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.CUSTOMER)).getSavedEntity(); + Mockito.reset(entityActionService); + Customer importedCustomer = (Customer) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), any(), eq(ActionType.ADDED), isNull()); - importEntity(tenantAdmin2, entitiesExportData.get(EntityType.CUSTOMER)); - verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + Mockito.reset(entityActionService); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + + EntityExportData updatedCustomerEntity = getAndClone(entitiesExportData, EntityType.CUSTOMER); + updatedCustomerEntity.getEntity().setEmail("t" + updatedCustomerEntity.getEntity().getEmail()); + Customer updatedCustomer = importEntity(tenantAdmin2, updatedCustomerEntity).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(updatedCustomer), any(), eq(ActionType.UPDATED), isNull()); verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); - Asset importedAsset = (Asset) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)).getSavedEntity(); + Mockito.reset(entityActionService); + + Asset importedAsset = (Asset) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.ASSET)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), any(), eq(ActionType.ADDED), isNull()); importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)); - verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + + + EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); + updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); + Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); + + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(updatedAsset), any(), eq(ActionType.UPDATED), isNull()); verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); - RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.RULE_CHAIN)).getSavedEntity(); + RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.RULE_CHAIN)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), any(), eq(ActionType.ADDED), isNull()); verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); - Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DASHBOARD)).getSavedEntity(); + Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DASHBOARD)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), any(), eq(ActionType.ADDED), isNull()); - DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE_PROFILE)).getSavedEntity(); + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE_PROFILE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), any(), eq(ActionType.ADDED), isNull()); verify(tbClusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); @@ -488,12 +513,17 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); - Device importedDevice = (Device) importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); + Device importedDevice = (Device) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), any(), eq(ActionType.ADDED), isNull()); verify(tbClusterService).onDeviceUpdated(eq(importedDevice), isNull()); - importEntity(tenantAdmin2, entitiesExportData.get(EntityType.DEVICE)); - verify(tbClusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); + verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); + updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); + Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); + verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); } @Test diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 239777f827..1c6f4c060a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -31,7 +31,7 @@ public class EntityImportResult> private ThrowingRunnable saveReferencesCallback = () -> {}; private ThrowingRunnable sendEventsCallback = () -> {}; - private Boolean updatedAllExternalIds; + private boolean updatedAllExternalIds = true; private boolean created; private boolean updated; diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1a3a0d02ff..d3e55f3cb9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -301,8 +301,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } } } - Collections.sort(ruleChainMetaData.getConnections(), Comparator.comparingInt(NodeConnectionInfo::getFromIndex) - .thenComparing(NodeConnectionInfo::getToIndex).thenComparing(NodeConnectionInfo::getType)); + if (ruleChainMetaData.getConnections() != null) { + Collections.sort(ruleChainMetaData.getConnections(), Comparator.comparingInt(NodeConnectionInfo::getFromIndex) + .thenComparing(NodeConnectionInfo::getToIndex).thenComparing(NodeConnectionInfo::getType)); + } return ruleChainMetaData; } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index d1ac360d44..8acd638ca4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -1109,7 +1109,7 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntityActiveSchedule = new AttributeKvEntity(); attributeKvEntityActiveSchedule.setId(compositeKeyActiveSchedule); attributeKvEntityActiveSchedule.setJsonValue( - "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":8.64e+7,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" + "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" ); attributeKvEntityActiveSchedule.setLastUpdateTs(0L); @@ -1166,6 +1166,8 @@ public class TbDeviceProfileNodeTest { TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); +// Mockito.reset(ctx); + node.onMsg(ctx, msg); verify(ctx).tellSuccess(msg); verify(ctx).enqueueForTellNext(theMsg, "Alarm Created"); From 5cad77910fb48db4a50e856e22a6c1a8fbc17201 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Sat, 18 Jun 2022 10:48:02 +0300 Subject: [PATCH 259/262] Improved method that replaces uuids --- .../impl/BaseEntityExportService.java | 13 +++- .../impl/DashboardExportService.java | 16 ++--- .../impl/RuleChainExportService.java | 4 +- .../impl/DashboardImportService.java | 2 +- .../impl/RuleChainImportService.java | 4 +- .../server/common/data/Dashboard.java | 28 ++++++++- .../thingsboard/common/util/JacksonUtil.java | 61 +++++++++++++++++++ .../thingsboard/common/util}/RegexUtils.java | 2 +- 8 files changed, 110 insertions(+), 20 deletions(-) rename {application/src/main/java/org/thingsboard/server/utils => common/util/src/main/java/org/thingsboard/common/util}/RegexUtils.java (96%) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index f233672297..169b7f273c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.sync.ie.exporting.impl; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -32,12 +34,19 @@ public abstract class BaseEntityExportService ctx, E mainEntity, D exportData) {} + protected void setRelatedEntities(EntitiesExportCtx ctx, E mainEntity, D exportData) { + } protected D newExportData() { return (D) new EntityExportData(); - }; + } + + ; public abstract Set getSupportedEntityTypes(); + protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { + JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid)); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 391606be63..91bb9ad8cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -27,9 +27,10 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; -import org.thingsboard.server.utils.RegexUtils; +import org.thingsboard.common.util.RegexUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.Set; import java.util.UUID; @@ -46,17 +47,12 @@ public class DashboardExportService extends BaseEntityExportService fields = Lists.newArrayList(entityAlias.fieldNames()); - for (String field : fields) { - if (field.equals("id")) continue; - JsonNode oldFieldValue = entityAlias.get(field); - JsonNode newFieldValue = JacksonUtil.toJsonNode(RegexUtils.replace(oldFieldValue.toString(), RegexUtils.UUID_PATTERN, uuid -> { - return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); - })); - ((ObjectNode) entityAlias).set(field, newFieldValue); - } + replaceUuidsRecursively(ctx, entityAlias, Collections.singleton("id")); } } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id")); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index a28f8b8847..6cebfe4757 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; -import org.thingsboard.server.utils.RegexUtils; +import org.thingsboard.common.util.RegexUtils; import java.util.Collections; import java.util.Optional; @@ -52,7 +52,7 @@ public class RuleChainExportService extends BaseEntityExportService { + String newRuleNodeConfigJson = RegexUtils.replace(JacksonUtil.toString(ruleNodeConfig), RegexUtils.UUID_PATTERN, uuid -> { return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); }); ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 4d767a5f61..77e1675abc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; -import org.thingsboard.server.utils.RegexUtils; +import org.thingsboard.common.util.RegexUtils; import java.util.ArrayList; import java.util.Arrays; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index a9e230a531..908d491a73 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -37,7 +37,7 @@ import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; -import org.thingsboard.server.utils.RegexUtils; +import org.thingsboard.common.util.RegexUtils; import java.util.Arrays; import java.util.Collections; @@ -95,7 +95,7 @@ public class RuleChainImportService extends BaseEntityImportService { JsonNode ruleNodeConfig = ruleNode.getConfiguration(); - String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { + String newRuleNodeConfigJson = RegexUtils.replace(JacksonUtil.toString(ruleNodeConfig), RegexUtils.UUID_PATTERN, uuid -> { return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFinalImportAttempt(), HINTS) .map(entityId -> entityId.getId().toString()) .orElse(uuid); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index b1ca86de10..c3e9ae561e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; @@ -24,7 +25,11 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; +import java.util.stream.StreamSupport; @EqualsAndHashCode(callSuper = true) public class Dashboard extends DashboardInfo implements ExportableEntity { @@ -33,7 +38,8 @@ public class Dashboard extends DashboardInfo implements ExportableEntity config.get("entityAliases")) .filter(JsonNode::isObject).orElse(null); } + @JsonIgnore + public List getWidgetsConfig() { + return Optional.ofNullable(configuration) + .map(config -> config.get("widgets")) + .filter(node -> !node.isEmpty()) + .map(node -> (ObjectNode) node) + .map(object -> { + List widgets = new ArrayList<>(object.size()); + object.forEach(child -> { + if (child.isObject()) { + widgets.add((ObjectNode) child); + } + }); + return widgets; + }) + .orElse(Collections.emptyList()); + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index d31eb2ee74..9c4b68d3c0 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -22,10 +22,16 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.UnaryOperator; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -151,4 +157,59 @@ public class JacksonUtil { + value + " cannot be transformed to a String", e); } } + + + public static JsonNode getSafely(JsonNode node, String... path) { + if (node == null) { + return null; + } + for (String p : path) { + if (!node.has(p)) { + return null; + } else { + node = node.get(p); + } + } + return node; + } + + public static void replaceUuidsRecursively(JsonNode node, Set skipFieldsSet, UnaryOperator replacer) { + if (node == null) { + return; + } + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + List fieldNames = new ArrayList<>(objectNode.size()); + objectNode.fieldNames().forEachRemaining(fieldNames::add); + for (String fieldName : fieldNames) { + if (skipFieldsSet.contains(fieldName)) { + continue; + } + var child = objectNode.get(fieldName); + if (child.isObject() || child.isArray()) { + replaceUuidsRecursively(child, skipFieldsSet, replacer); + } else if (child.isTextual()) { + String text = child.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + objectNode.put(fieldName, newText); + } + } + } + } else if (node.isArray()) { + ArrayNode array = (ArrayNode) node; + for (int i = 0; i < array.size(); i++) { + JsonNode arrayElement = array.get(i); + if (arrayElement.isObject() || arrayElement.isArray()) { + replaceUuidsRecursively(arrayElement, skipFieldsSet, replacer); + } else if (arrayElement.isTextual()) { + String text = arrayElement.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + array.set(i, newText); + } + } + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/utils/RegexUtils.java b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/utils/RegexUtils.java rename to common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java index 60fe870d69..af968a6bff 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RegexUtils.java +++ b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.common.util; import lombok.AccessLevel; import lombok.NoArgsConstructor; From bdd843204936001cc8be69218f182348c3e19f27 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Jun 2022 12:00:21 +0300 Subject: [PATCH 260/262] Fix import/export of rule chains and dashboard with external id references --- .../impl/DashboardExportService.java | 6 ++---- .../impl/RuleChainExportService.java | 7 +------ .../impl/BaseEntityImportService.java | 9 ++++++++ .../impl/DashboardImportService.java | 21 ++++++------------- .../impl/RuleChainImportService.java | 11 +--------- .../server/common/data/Dashboard.java | 13 +++++++----- 6 files changed, 27 insertions(+), 40 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 91bb9ad8cd..9a3b195b57 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -45,10 +45,8 @@ public class DashboardExportService extends BaseEntityExportService { - return getExternalIdOrElseInternalByUuid(ctx, UUID.fromString(uuid)).toString(); - }); - ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); - ruleNode.setConfiguration(ruleNodeConfig); + replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet()); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 31583e0ee9..26c5a869f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.fasterxml.jackson.databind.JsonNode; import com.google.api.client.util.Objects; import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; @@ -23,6 +24,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -53,11 +55,13 @@ import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -386,4 +390,9 @@ public abstract class BaseEntityImportService skipFieldsSet, LinkedHashSet hints) { + JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet, + uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid)); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 77e1675abc..b6de84bfff 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.dashboard.DashboardService; @@ -72,21 +73,11 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - JsonNode configuration = dashboard.getConfiguration(); - JsonNode entityAliases = configuration.get("entityAliases"); - if (entityAliases != null && entityAliases.isObject()) { - for (JsonNode entityAlias : entityAliases) { - ArrayList fields = Lists.newArrayList(entityAlias.fieldNames()); - for (String field : fields) { - if (field.equals("id")) continue; - JsonNode oldFieldValue = entityAlias.get(field); - JsonNode newFieldValue = JacksonUtil.toJsonNode(RegexUtils.replace(oldFieldValue.toString(), RegexUtils.UUID_PATTERN, uuid -> { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFinalImportAttempt(), HINTS) - .map(entityId -> entityId.getId().toString()).orElse(uuid); - })); - ((ObjectNode) entityAlias).set(field, newFieldValue); - } - } + for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { + replaceIdsRecursively(ctx, idProvider, entityAlias, Collections.emptySet(), HINTS); + } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"), HINTS); } return dashboard; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 908d491a73..916c10844f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -93,16 +93,7 @@ public class RuleChainImportService extends BaseEntityImportService { - JsonNode ruleNodeConfig = ruleNode.getConfiguration(); - String newRuleNodeConfigJson = RegexUtils.replace(JacksonUtil.toString(ruleNodeConfig), RegexUtils.UUID_PATTERN, uuid -> { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFinalImportAttempt(), HINTS) - .map(entityId -> entityId.getId().toString()) - .orElse(uuid); - }); - ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); - ruleNode.setConfiguration(ruleNodeConfig); - }); + ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS)); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index c3e9ae561e..049c830745 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -73,16 +73,19 @@ public class Dashboard extends DashboardInfo implements ExportableEntity config.get("entityAliases")) - .filter(JsonNode::isObject).orElse(null); + public List getEntityAliasesConfig() { + return getChildObjects("entityAliases"); } @JsonIgnore public List getWidgetsConfig() { + return getChildObjects("widgets"); + } + + @JsonIgnore + private List getChildObjects(String propertyName) { return Optional.ofNullable(configuration) - .map(config -> config.get("widgets")) + .map(config -> config.get(propertyName)) .filter(node -> !node.isEmpty()) .map(node -> (ObjectNode) node) .map(object -> { From 6bf6893b308fede833955de5277be6595dc03623 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 20 Jun 2022 16:53:06 +0300 Subject: [PATCH 261/262] Revert get entity alarms controller method --- .../org/thingsboard/server/controller/AlarmController.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index b0ed1ed01d..939e9da8a0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -226,11 +226,7 @@ public class AlarmController extends BaseController { TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); try { - if (getCurrentUser().isCustomerUser()) { - return checkNotNull(alarmService.findCustomerAlarms(getCurrentUser().getTenantId(), getCurrentUser().getCustomerId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } else { - return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } + return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); } catch (Exception e) { throw handleException(e); } From 8fa5fc8c3fda3bde9ace43101323cf608a868dda Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Jun 2022 17:09:05 +0300 Subject: [PATCH 262/262] Improved rate limits processing filter to avoid NPE --- .../server/config/RateLimitProcessingFilter.java | 11 ++++++++++- .../service/install/update/RateLimitsUpdater.java | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 1415ff4ece..86675a79a8 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.config; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -41,6 +43,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +@Slf4j @Component public class RateLimitProcessingFilter extends GenericFilterBean { @@ -58,7 +61,13 @@ public class RateLimitProcessingFilter extends GenericFilterBean { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultProfileConfiguration(); + var profile = tenantProfileCache.get(user.getTenantId()); + if (profile == null) { + log.debug("[{}] Failed to lookup tenant profile", user.getTenantId()); + errorResponseHandler.handle(new BadCredentialsException("Failed to lookup tenant profile"), (HttpServletResponse) response); + return; + } + var profileConfiguration = profile.getDefaultProfileConfiguration(); if (!checkRateLimits(user.getTenantId(), profileConfiguration.getTenantServerRestLimitsConfiguration(), perTenantLimits, response)) { return; } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index 054f68f71d..c136f97b52 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.