diff --git a/application/pom.xml b/application/pom.xml index 6ca5822d21..91dec1847e 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -193,8 +193,8 @@ logback-classic - javax.mail - mail + com.sun.mail + javax.mail org.apache.curator diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 32cecf6e4e..9805c6f996 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -17,7 +17,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", "name": "Is Thermostat?", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" } @@ -113,7 +113,7 @@ }, "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", "name": "Relate to Asset", - "debugMode": true, + "debugMode": false, "configuration": { "direction": "FROM", "relationType": "ToAlarmPropagationAsset", diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json index 8fd10c88f1..d67052cbc5 100644 --- a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -81,7 +81,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", "name": "Check Alarms", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" } @@ -93,7 +93,7 @@ }, "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", "name": "Fetch Configuration", - "debugMode": true, + "debugMode": false, "configuration": { "clientAttributeNames": [], "sharedAttributeNames": [], diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 4e3d69c2b7..61c5c0da28 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -163,7 +163,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes, String failureMessage) { try { - checkActive(); + checkActive(msg); EntityId entityId = msg.getOriginator(); TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); List relations = nodeRoutes.get(originatorNodeId).stream() @@ -272,6 +278,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor { + private final String ruleChainName; private final RuleChainId ruleChainId; - private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) { super(systemContext, tenantId, ruleNodeId); + this.ruleChainName = ruleChainName; this.ruleChainId = ruleChainId; - setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext, + setProcessor(new RuleNodeActorMessageProcessor(tenantId, this.ruleChainName, ruleNodeId, systemContext, context().parent(), context().self())); } @@ -96,19 +98,21 @@ public class RuleNodeActor extends ComponentActor { - private final ActorRef parent; + private final String ruleChainName; private final ActorRef self; - private final RuleChainService service; private RuleNode ruleNode; private TbNode tbNode; private DefaultTbContext defaultCtx; - RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext + RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { super(systemContext, tenantId, ruleNodeId); - this.parent = parent; + this.ruleChainName = ruleChainName; this.self = self; - this.service = systemContext.getRuleChainService(); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode)); } @@ -63,8 +59,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -74,11 +77,17 @@ public abstract class ComponentMsgProcessor extends Abstract schedulePeriodicMsgWithDelay(context, new StatsPersistTick(), statsPersistFrequency, statsPersistFrequency); } - protected void checkActive() { + protected void checkActive(TbMsg tbMsg) throws RuleNodeException { if (state != ComponentLifecycleState.ACTIVE) { log.debug("Component is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId); - throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId); + RuleNodeException ruleNodeException = getInactiveException(); + if (tbMsg != null) { + tbMsg.getCallback().onFailure(ruleNodeException); + } + throw ruleNodeException; } } + abstract protected RuleNodeException getInactiveException(); + } diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 3097b9c2b1..49cfe482df 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -338,7 +338,7 @@ public class AuthController extends BaseController { @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) @ResponseBody - public List getOath2Clients() throws ThingsboardException { + public List getOAuth2Clients() throws ThingsboardException { try { return oauth2Service.getOAuth2Clients(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a5abd77ad5..b7f6bd3dd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -45,6 +46,7 @@ import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; @@ -100,7 +102,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.tell(actorMsg.get(), ActorRef.noSender()); + TbActorMsg tbActorMsg = actorMsg.get(); + if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) { + tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg); + } else { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); + } } callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 0c48c54412..182a62d397 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -48,6 +50,9 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService; import javax.annotation.PostConstruct; @@ -81,6 +86,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; private final TbQueueRuleEngineSettings ruleEngineSettings; private final RuleEngineStatisticsService statisticsService; + private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); @@ -90,13 +96,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbRuleEngineSubmitStrategyFactory submitStrategyFactory, TbQueueRuleEngineSettings ruleEngineSettings, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, - ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { + ActorSystemContext actorContext, DataDecodingEncodingService encodingService, + TbRuleEngineDeviceRpcService tbDeviceRpcService) { super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); this.statisticsService = statisticsService; this.ruleEngineSettings = ruleEngineSettings; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.submitStrategyFactory = submitStrategyFactory; this.processingStrategyFactory = processingStrategyFactory; + this.tbDeviceRpcService = tbDeviceRpcService; } @PostConstruct @@ -227,7 +235,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); + } else if (nfMsg.hasFromDeviceRpcResponse()) { + TransportProtos.FromDeviceRPCResponseProto proto = nfMsg.getFromDeviceRpcResponse(); + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbDeviceRpcService.processRpcResponseFromDevice(response); + callback.onSuccess(); } else { + log.trace("Received notification with missing handler"); callback.onSuccess(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index c2705fcbdc..4007c9e17d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,7 +23,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java index d0b1f7f99a..b9741d2433 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -23,7 +23,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -77,8 +76,8 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS } } int submitSize = pendingPack.size(); - if (log.isInfoEnabled() && submitSize > 0) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + if (log.isDebugEnabled() && submitSize > 0) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize); } pendingPack.forEach(msgConsumer); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java index ffd1dd49d1..3420933d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -19,14 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { @@ -37,8 +31,8 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS @Override public void submitAttempt(BiConsumer> msgConsumer) { - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); } orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java index ae5993cb1c..473810b86c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -15,26 +15,18 @@ */ package org.thingsboard.server.service.queue.processing; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; -import java.util.stream.Collectors; @Slf4j public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index b258c6db1b..37e9419edd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -30,6 +30,5 @@ public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialBy @Override protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java index ef45b983fc..125a1d8ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @@ -63,8 +61,8 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu if (idx < listSize) { IdMsgPair pair = orderedMsgList.get(idx); expectedMsgId = pair.uuid; - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg); } msgConsumer.accept(pair.uuid, pair.msg); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index bbf283e962..80b0523a81 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -82,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory { retryCount++; double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); return new TbRuleEngineProcessingDecision(true, null); } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); @@ -98,7 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory { if (retrySuccessful) { result.getSuccessMap().forEach(toReprocess::put); } - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } @@ -126,7 +126,7 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { if (!result.isSuccess()) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); } if (log.isTraceEnabled()) { result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java index 935f7f5e3a..1d44a772aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java @@ -51,9 +51,9 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey()); oauth2User.setFirstName(firstName); } - if (!StringUtils.isEmpty(config.getBasic().getCustomerNameStrategyPattern())) { + if (!StringUtils.isEmpty(config.getBasic().getCustomerNamePattern())) { StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); - String customerName = sub.replace(config.getBasic().getCustomerNameStrategyPattern()); + String customerName = sub.replace(config.getBasic().getCustomerNamePattern()); oauth2User.setCustomerName(customerName); } return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); @@ -68,7 +68,7 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen return email.substring(email .indexOf("@") + 1); case CUSTOM_TENANT_STRATEGY: StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX); - return sub.replace(config.getBasic().getTenantNameStrategyPattern()); + return sub.replace(config.getBasic().getTenantNamePattern()); default: throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!"); } @@ -78,7 +78,6 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen String result = null; try { result = (String) attributes.get(key); - } catch (Exception e) { log.warn("Can't convert attribute to String by key " + key); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java index 31e2c2bbec..832a1cd39b 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -41,7 +41,7 @@ public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); } - public OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { + private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { if (!StringUtils.isEmpty(custom.getUsername()) && !StringUtils.isEmpty(custom.getPassword())) { restTemplateBuilder = restTemplateBuilder.basicAuthentication(custom.getUsername(), custom.getPassword()); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5c663de996..4ed2838ca8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -126,8 +126,8 @@ security: firstNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_FIRST_NAME_ATTRIBUTE_KEY:}" lastNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_LAST_NAME_ATTRIBUTE_KEY:}" tenantNameStrategy: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_STRATEGY:domain}" # domain, email or custom - tenantNameStrategyPattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_STRATEGY_PATTERN:}" - customerNameStrategyPattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_CUSTOMER_NAME_STRATEGY_PATTERN:}" + tenantNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_PATTERN:}" # %{attribute_key} as placeholder for attributes value by key + customerNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_CUSTOMER_NAME_PATTERN:}" # %{attribute_key} as placeholder for attributes value by key custom: url: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_URL:}" username: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_USERNAME:}" @@ -637,7 +637,7 @@ queue: pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -657,7 +657,7 @@ queue: pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" queues: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 3edf4e5061..9da7407552 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import java.io.IOException; import java.io.Serializable; import java.util.UUID; @@ -47,7 +48,7 @@ public final class TbMsg implements Serializable { private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; //This field is not serialized because we use queues and there is no need to do it - private final TbMsgCallback callback; + transient private final TbMsgCallback callback; public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); @@ -156,4 +157,13 @@ public final class TbMsg implements Serializable { public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); } + + public TbMsgCallback getCallback() { + //May be null in case of deserialization; + if (callback != null) { + return callback; + } else { + return TbMsgCallback.EMPTY; + } + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java index 3f437dd1d7..288e1b6002 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -36,9 +36,15 @@ public class RuleNodeException extends RuleEngineException { public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { super(message); this.ruleChainName = ruleChainName; - this.ruleNodeName = ruleNode.getName(); - this.ruleChainId = ruleNode.getRuleChainId(); - this.ruleNodeId = ruleNode.getId(); + if (ruleNode != null) { + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } else { + ruleNodeName = "Unknown"; + ruleChainId = new RuleChainId(RuleChainId.NULL_UUID); + ruleNodeId = new RuleNodeId(RuleNodeId.NULL_UUID); + } } public String toJsonString() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index f108de4a43..3355c55d7e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -67,6 +67,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { try { QueueDescription queueDescription = new QueueDescription(topic); + queueDescription.setRequiresDuplicateDetection(false); setQueueConfigs(queueDescription); client.createQueue(queueDescription); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java index cca599d59a..734c79d927 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -31,9 +31,9 @@ import org.apache.qpid.proton.amqp.transport.SenderSettleMode; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.time.Duration; @@ -50,102 +50,72 @@ import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbServiceBusConsumerTemplate implements TbQueueConsumer { +public class TbServiceBusConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbServiceBusSettings serviceBusSettings; private final Gson gson = new Gson(); private Set receivers; - private volatile Set partitions; - private volatile boolean subscribed; - private volatile boolean stopped = false; - private Map> pendingMessages = new ConcurrentHashMap<>(); + private final Map> pendingMessages = new ConcurrentHashMap<>(); private volatile int messagesPerQueue; public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.serviceBusSettings = serviceBusSettings; } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", getTopic()); + } else { + log.error("Failed to receive messages", e); + } + return Collections.emptyList(); + } } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); } @Override - public void unsubscribe() { - stopped = true; + protected void doUnsubscribe() { receivers.forEach(CoreMessageReceiver::closeAsync); } - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - createReceivers(); - messagesPerQueue = receivers.size() / partitions.size(); - subscribed = true; - } - - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to parse message.", e); - throw new RuntimeException("Failed to parse message.", e); - } - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - private void createReceivers() { List> receiverFutures = partitions.stream() .map(TopicPartitionInfo::getFullTopicName) @@ -167,7 +137,7 @@ public class TbServiceBusConsumerTemplate implements TbQue receivers = new HashSet<>(fromList(receiverFutures).get()); } catch (InterruptedException | ExecutionException e) { if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); + log.info("[{}] Service Bus consumer is stopped.", getTopic()); } else { log.error("Failed to create receivers", e); } @@ -196,13 +166,7 @@ public class TbServiceBusConsumerTemplate implements TbQue } @Override - public void commit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + protected T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java index 5d9a931378..3c5228198d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,14 +43,14 @@ public class TbServiceBusProducerTemplate implements TbQue private final Gson gson = new Gson(); private final TbQueueAdmin admin; private final TbServiceBusSettings serviceBusSettings; - private final Map clients = new HashMap<>(); - private ExecutorService executorService; + private final Map clients = new ConcurrentHashMap<>(); + private final ExecutorService executorService; public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { this.admin = admin; this.defaultTopic = defaultTopic; this.serviceBusSettings = serviceBusSettings; - executorService = Executors.newSingleThreadExecutor(); + executorService = Executors.newCachedThreadPool(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..bb83a79250 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class AbstractParallelTbQueueConsumerTemplate extends AbstractTbQueueConsumerTemplate { + + protected ListeningExecutorService consumerExecutor; + + public AbstractParallelTbQueueConsumerTemplate(String topic) { + super(topic); + } + + protected void initNewExecutor(int threadPoolSize) { + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + try { + consumerExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + log.trace("Interrupted while waiting for consumer executor to stop"); + } + } + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadPoolSize)); + } + + protected void shutdownExecutor() { + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..c8cc545601 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -0,0 +1,152 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { + + private volatile boolean subscribed; + protected volatile boolean stopped = false; + protected volatile Set partitions; + protected final Lock consumerLock = new ReentrantLock(); + + @Getter + private final String topic; + + public AbstractTbQueueConsumerTemplate(String topic) { + this.topic = topic; + } + + @Override + public void subscribe() { + consumerLock.lock(); + try { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public void subscribe(Set partitions) { + consumerLock.lock(); + try { + this.partitions = partitions; + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + long pollStartTs = System.currentTimeMillis(); + consumerLock.lock(); + try { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + doSubscribe(topicNames); + subscribed = true; + } + + List records = doPoll(durationInMillis); + if (!records.isEmpty()) { + List result = new ArrayList<>(records.size()); + records.forEach(record -> { + try { + if (record != null) { + result.add(decode(record)); + } + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + throw new RuntimeException("Failed to decode record: ", e); + } + }); + return result; + } else { + long pollDuration = System.currentTimeMillis() - pollStartTs; + if (pollDuration < durationInMillis) { + try { + Thread.sleep(durationInMillis - pollDuration); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + } + } + } finally { + consumerLock.unlock(); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + consumerLock.lock(); + try { + doCommit(); + } finally { + consumerLock.unlock(); + } + } + + @Override + public void unsubscribe() { + stopped = true; + consumerLock.lock(); + try { + doUnsubscribe(); + } finally { + consumerLock.unlock(); + } + } + + abstract protected List doPoll(long durationInMillis); + + abstract protected T decode(R record) throws IOException; + + abstract protected void doSubscribe(List topicNames); + + abstract protected void doCommit(); + + abstract protected void doUnsubscribe(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 6b1c051eeb..1b5619fd26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -16,16 +16,14 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; @@ -33,26 +31,16 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. */ @Slf4j -public class TbKafkaConsumerTemplate implements TbQueueConsumer { +public class TbKafkaConsumerTemplate extends AbstractTbQueueConsumerTemplate, T> { private final TbQueueAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; - private volatile boolean subscribed; - private volatile Set partitions; - private final Lock consumerLock; - - @Getter - private final String topic; @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, @@ -60,6 +48,7 @@ public class TbKafkaConsumerTemplate implements TbQueueCon boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords, TbQueueAdmin admin) { + super(topic); Properties props = settings.toProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); if (groupId != null) { @@ -75,94 +64,42 @@ public class TbKafkaConsumerTemplate implements TbQueueCon this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; - this.topic = topic; - this.consumerLock = new ReentrantLock(); } @Override - public void subscribe() { - consumerLock.lock(); - try { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } finally { - consumerLock.unlock(); - } + protected void doSubscribe(List topicNames) { + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); } @Override - public void subscribe(Set partitions) { - consumerLock.lock(); - try { - this.partitions = partitions; - subscribed = false; - } finally { - consumerLock.unlock(); + protected List> doPoll(long durationInMillis) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.isEmpty()) { + return Collections.emptyList(); + } else { + List> recordList = new ArrayList<>(256); + records.forEach(recordList::add); + return recordList; } } @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - consumerLock.lock(); - try { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; - } - } finally { - consumerLock.unlock(); - } - } - return Collections.emptyList(); + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(new KafkaTbQueueMsg(record)); } @Override - public void commit() { - consumerLock.lock(); - try { - consumer.commitAsync(); - } finally { - consumerLock.unlock(); - } + protected void doCommit() { + consumer.commitAsync(); } @Override - public void unsubscribe() { - consumerLock.lock(); - try { - if (consumer != null) { - consumer.unsubscribe(); - consumer.close(); - } - } finally { - consumerLock.unlock(); + protected void doUnsubscribe() { + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); } } - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(new KafkaTbQueueMsg(record)); - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index 1241370d05..d0a514ffd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.pubsub; +import com.google.api.gax.rpc.AlreadyExistsException; import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; @@ -24,9 +25,9 @@ import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -103,7 +104,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String partition) { - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + TopicName topicName = TopicName.newBuilder() + .setTopic(partition) + .setProject(pubSubSettings.getProjectId()) + .build(); if (topicSet.contains(topicName.toString())) { createSubscriptionIfNotExists(partition, topicName); @@ -121,13 +125,18 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); + try { + topicAdminClient.createTopic(topicName); + log.info("Created new topic: [{}]", topicName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Topic already exist.", topicName.toString()); + } finally { + topicSet.add(topicName.toString()); + } createSubscriptionIfNotExists(partition, topicName); } - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + private void createSubscriptionIfNotExists(String partition, TopicName topicName) { ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); @@ -153,9 +162,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { setAckDeadline(subscriptionBuilder); setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); + try { + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Subscription already exist.", subscriptionName.toString()); + } finally { + subscriptionSet.add(subscriptionName.toString()); + } } private void setAckDeadline(Subscription.Builder builder) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 5dc795739a..b5b6126cd5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -30,27 +30,25 @@ import com.google.pubsub.v1.PullResponse; import com.google.pubsub.v1.ReceivedMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.stream.Collectors; @Slf4j -public class TbPubSubConsumerTemplate implements TbQueueConsumer { +public class TbPubSubConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; @@ -58,23 +56,18 @@ public class TbPubSubConsumerTemplate implements TbQueueCo private final TbQueueMsgDecoder decoder; private final TbPubSubSettings pubSubSettings; - private volatile boolean subscribed; - private volatile Set partitions; private volatile Set subscriptionNames; private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - private ExecutorService consumerExecutor; private final SubscriberStub subscriber; - private volatile boolean stopped; - private volatile int messagesPerTopic; public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.pubSubSettings = pubSubSettings; this.topic = topic; this.decoder = decoder; - try { SubscriberStubSettings subscriberStubSettings = SubscriberStubSettings.newBuilder() @@ -84,89 +77,50 @@ public class TbPubSubConsumerTemplate implements TbQueueCo .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) .build(); - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { log.error("Failed to create subscriber.", e); throw new RuntimeException("Failed to create subscriber.", e); } - stopped = false; } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + try { + List messages = receiveMessages(); + if (!messages.isEmpty()) { + return messages.stream().map(ReceivedMessage::getMessage).collect(Collectors.toList()); + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + return Collections.emptyList(); } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + subscriptionNames = new LinkedHashSet<>(topicNames); + subscriptionNames.forEach(admin::createTopicIfNotExists); + initNewExecutor(subscriptionNames.size() + 1); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); } @Override - public void unsubscribe() { - stopped = true; - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - + protected void doUnsubscribe() { if (subscriber != null) { subscriber.close(); } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); - subscriptionNames.forEach(admin::createTopicIfNotExists); - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); - subscribed = true; - } - List messages; - try { - messages = receiveMessages(); - if (!messages.isEmpty()) { - List result = new ArrayList<>(); - messages.forEach(msg -> { - try { - result.add(decode(msg.getMessage())); - } catch (InvalidProtocolBufferException e) { - log.error("Failed decode record: [{}]", msg); - } - }); - return result; - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); - acknowledgeRequests.clear(); + shutdownExecutor(); } private List receiveMessages() throws ExecutionException, InterruptedException { @@ -175,7 +129,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo PullRequest pullRequest = PullRequest.newBuilder() .setMaxMessages(messagesPerTopic) - .setReturnImmediately(false) // return immediately if messages are not available +// .setReturnImmediately(false) // return immediately if messages are not available .setSubscription(subscriptionName) .build(); @@ -211,6 +165,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo return transform.get(); } + @Override public T decode(PubsubMessage message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 2cd2e1054e..ea713827e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -49,7 +49,7 @@ public class TbPubSubProducerTemplate implements TbQueuePr private final Map publisherMap = new ConcurrentHashMap<>(); - private ExecutorService pubExecutor = Executors.newCachedThreadPool(); + private final ExecutorService pubExecutor = Executors.newCachedThreadPool(); public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { this.defaultTopic = defaultTopic; @@ -124,8 +124,8 @@ public class TbPubSubProducerTemplate implements TbQueuePr publisherMap.put(topic, publisher); return publisher; } catch (IOException e) { - log.error("Failed to create topic [{}].", topic, e); - throw new RuntimeException("Failed to create topic.", e); + log.error("Failed to create Publisher for the topic [{}].", topic, e); + throw new RuntimeException("Failed to create Publisher for the topic.", e); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 3bef6deb84..676ee5354f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -27,13 +27,11 @@ import java.util.concurrent.TimeoutException; @Slf4j public class TbRabbitMqAdmin implements TbQueueAdmin { - private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; private final Map arguments; public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { - this.rabbitMqSettings = rabbitMqSettings; this.arguments = arguments; try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java index 25d7719163..45dc9d6a05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -23,9 +23,9 @@ import com.rabbitmq.client.GetResponse; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; @@ -37,33 +37,26 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @Slf4j -public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { +public class TbRabbitMqConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; - private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; - private volatile Set partitions; - private volatile boolean subscribed; private volatile Set queues; - private volatile boolean stopped; public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; - this.rabbitMqSettings = rabbitMqSettings; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); } catch (IOException | TimeoutException e) { log.error("Failed to create connection.", e); throw new RuntimeException("Failed to create connection.", e); } - try { channel = connection.createChannel(); } catch (IOException e) { @@ -74,25 +67,42 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } else { + return Collections.emptyList(); + } } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + queues.forEach(admin::createTopicIfNotExists); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } } @Override - public void unsubscribe() { - stopped = true; + protected void doUnsubscribe() { if (channel != null) { try { channel.close(); @@ -109,63 +119,6 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } } - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - - queues.forEach(admin::createTopicIfNotExists); - subscribed = true; - } - - List result = queues.stream() - .map(queue -> { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - }).filter(Objects::nonNull).map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to decode message: [{}].", message); - throw new RuntimeException("Failed to decode message.", e); - } - }).collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to wait.", e); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - public T decode(GetResponse message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java index 91b46213a5..a58b817f78 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java @@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; @@ -39,10 +41,12 @@ public class TbRabbitMqProducerTemplate implements TbQueue private final Gson gson = new Gson(); private final TbQueueAdmin admin; private final TbRabbitMqSettings rabbitMqSettings; - private ListeningExecutorService producerExecutor; + private final ListeningExecutorService producerExecutor; private final Channel channel; private final Connection connection; + private final Set topics = ConcurrentHashMap.newKeySet(); + public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { this.admin = admin; this.defaultTopic = defaultTopic; @@ -75,6 +79,7 @@ public class TbRabbitMqProducerTemplate implements TbQueue @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + createTopicIfNotExist(tpi); AMQP.BasicProperties properties = new AMQP.BasicProperties(); try { channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); @@ -110,4 +115,11 @@ public class TbRabbitMqProducerTemplate implements TbQueue } } + private void createTopicIfNotExist(TopicPartitionInfo tpi) { + if (topics.contains(tpi)) { + return; + } + admin.createTopicIfNotExists(tpi.getFullTopicName()); + topics.add(tpi); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 3e71388844..f4f279a02b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -25,21 +25,17 @@ import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,34 +43,28 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { +public class TbAwsSqsConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private static final int MAX_NUM_MSGS = 10; private final Gson gson = new Gson(); private final TbQueueAdmin admin; private final AmazonSQS sqsClient; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbAwsSqsSettings sqsSettings; private final List pendingMessages = new CopyOnWriteArrayList<>(); private volatile Set queueUrls; - private volatile Set partitions; - private ListeningExecutorService consumerExecutor; - private volatile boolean subscribed; - private volatile boolean stopped = false; public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.sqsSettings = sqsSettings; AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); @@ -87,81 +77,60 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo } @Override - public String getTopic() { - return topic; + protected void doSubscribe(List topicNames) { + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + initNewExecutor(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1); } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected List doPoll(long durationInMillis) { + int duration = (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis); + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, duration)) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", getTopic()); + } else { + log.error("Failed to pool messages.", e); + } + return Collections.emptyList(); + } } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + public T decode(Message message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); + return decoder.decode(msg); } @Override - public void unsubscribe() { - stopped = true; - - if (sqsClient != null) { - sqsClient.shutdown(); - } - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } + protected void doCommit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + pendingMessages.clear(); } @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); - subscribed = true; - } - - if (!pendingMessages.isEmpty()) { - log.warn("Present {} non committed messages.", pendingMessages.size()); - return Collections.emptyList(); - } - - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .map(msg -> { - try { - return decode(msg); - } catch (IOException e) { - log.error("Failed to decode message: [{}]", msg); - return null; - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", topic); - } else { - log.error("Failed to pool messages.", e); - } - } + protected void doUnsubscribe() { + stopped = true; + if (sqsClient != null) { + sqsClient.shutdown(); } - return Collections.emptyList(); + shutdownExecutor(); } private ListenableFuture> poll(String url, int waitTimeSeconds) { @@ -172,7 +141,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo ReceiveMessageRequest request = new ReceiveMessageRequest(); request .withWaitTimeSeconds(waitTimeSeconds) - .withMessageAttributeNames("headers") .withQueueUrl(url) .withMaxNumberOfMessages(MAX_NUM_MSGS); return sqsClient.receiveMessage(request).getMessages(); @@ -194,25 +162,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo }, consumerExecutor); } - @Override - public void commit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - - pendingMessages.clear(); - } - - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - @Data private static class AwsSqsMsgWrapper { private final String url; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 2d85539184..6110d08c5e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -80,7 +81,9 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + sendMsgRequest.withMessageGroupId(tpi.getTopic()); + sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); Futures.addCallback(future, new FutureCallback() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 70a9587bef..c6cbbfd256 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -55,7 +55,6 @@ public class TbAwsSqsQueueAttributes { @PostConstruct private void init() { defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); coreAttributes = getConfigs(coreProperties); ruleEngineAttributes = getConfigs(ruleEngineProperties); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 82678fde5d..3b4b26aa47 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -37,6 +37,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.lang.reflect.Field; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -55,6 +56,8 @@ public class CoapTransportResource extends CoapResource { private final Field observerField; private final long timeout; private final ConcurrentMap tokenToSessionIdMap = new ConcurrentHashMap<>(); + private final Set rpcSubscriptions = ConcurrentHashMap.newKeySet(); + private final Set attributeSubscriptions = ConcurrentHashMap.newKeySet(); public CoapTransportResource(CoapTransportContext context, String name) { super(name); @@ -149,11 +152,13 @@ public class CoapTransportResource extends CoapResource { transportService.process(sessionInfo, transportContext.getAdaptor().convertToPostAttributes(sessionId, request), new CoapOkCallback(exchange)); + reportActivity(sessionId, sessionInfo); break; case POST_TELEMETRY_REQUEST: transportService.process(sessionInfo, transportContext.getAdaptor().convertToPostTelemetry(sessionId, request), new CoapOkCallback(exchange)); + reportActivity(sessionId, sessionInfo); break; case CLAIM_REQUEST: transportService.process(sessionInfo, @@ -161,6 +166,7 @@ public class CoapTransportResource extends CoapResource { new CoapOkCallback(exchange)); break; case SUBSCRIBE_ATTRIBUTES_REQUEST: + attributeSubscriptions.add(sessionId); advanced.setObserver(new CoapExchangeObserverProxy((ExchangeObserver) observerField.get(advanced), registerAsyncCoapSession(exchange, request, sessionInfo, sessionId))); transportService.process(sessionInfo, @@ -168,6 +174,7 @@ public class CoapTransportResource extends CoapResource { new CoapNoOpCallback(exchange)); break; case UNSUBSCRIBE_ATTRIBUTES_REQUEST: + attributeSubscriptions.remove(sessionId); TransportProtos.SessionInfoProto attrSession = lookupAsyncSessionInfo(request); if (attrSession != null) { transportService.process(attrSession, @@ -177,6 +184,7 @@ public class CoapTransportResource extends CoapResource { } break; case SUBSCRIBE_RPC_COMMANDS_REQUEST: + rpcSubscriptions.add(sessionId); advanced.setObserver(new CoapExchangeObserverProxy((ExchangeObserver) observerField.get(advanced), registerAsyncCoapSession(exchange, request, sessionInfo, sessionId))); transportService.process(sessionInfo, @@ -184,13 +192,13 @@ public class CoapTransportResource extends CoapResource { new CoapNoOpCallback(exchange)); break; case UNSUBSCRIBE_RPC_COMMANDS_REQUEST: + rpcSubscriptions.remove(sessionId); TransportProtos.SessionInfoProto rpcSession = lookupAsyncSessionInfo(request); if (rpcSession != null) { transportService.process(rpcSession, TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(), new CoapOkCallback(exchange)); - transportService.process(sessionInfo, getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); - transportService.deregisterSession(rpcSession); + closeAndDeregister(sessionInfo); } break; case TO_DEVICE_RPC_RESPONSE: @@ -221,6 +229,14 @@ public class CoapTransportResource extends CoapResource { })); } + private void reportActivity(UUID sessionId, TransportProtos.SessionInfoProto sessionInfo) { + transportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(attributeSubscriptions.contains(sessionId)) + .setRpcSubscription(rpcSubscriptions.contains(sessionId)) + .setLastActivityTime(System.currentTimeMillis()) + .build(), TransportServiceCallback.EMPTY); + } + private TransportProtos.SessionInfoProto lookupAsyncSessionInfo(Request request) { String token = request.getSource().getHostAddress() + ":" + request.getSourcePort() + ":" + request.getTokenString(); return tokenToSessionIdMap.remove(token); @@ -438,6 +454,9 @@ public class CoapTransportResource extends CoapResource { private void closeAndDeregister(TransportProtos.SessionInfoProto session) { transportService.process(session, getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); transportService.deregisterSession(session); + UUID sessionId = new UUID(session.getSessionIdMSB(), session.getSessionIdLSB()); + rpcSubscriptions.remove(sessionId); + attributeSubscriptions.remove(sessionId); } } diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index c2f171e39b..727600a626 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; @@ -102,6 +103,7 @@ public class DeviceApiController { TransportService transportService = transportContext.getTransportService(); transportService.process(sessionInfo, JsonConverter.convertToAttributesProto(new JsonParser().parse(json)), new HttpOkCallback(responseWriter)); + reportActivity(sessionInfo); })); return responseWriter; } @@ -115,6 +117,7 @@ public class DeviceApiController { TransportService transportService = transportContext.getTransportService(); transportService.process(sessionInfo, JsonConverter.convertToTelemetryProto(new JsonParser().parse(json)), new HttpOkCallback(responseWriter)); + reportActivity(sessionInfo); })); return responseWriter; } @@ -274,7 +277,6 @@ public class DeviceApiController { } } - private static class HttpSessionListener implements SessionMsgListener { private final DeferredResult responseWriter; @@ -308,4 +310,13 @@ public class DeviceApiController { responseWriter.setResult(new ResponseEntity<>(JsonConverter.toJson(msg).toString(), HttpStatus.OK)); } } + + private void reportActivity(SessionInfoProto sessionInfo) { + transportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(false) + .setRpcSubscription(false) + .setLastActivityTime(System.currentTimeMillis()) + .build(), TransportServiceCallback.EMPTY); + } + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportServiceCallback.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportServiceCallback.java index 1f0d6c4c2c..f5b9ff22dd 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportServiceCallback.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportServiceCallback.java @@ -20,7 +20,20 @@ package org.thingsboard.server.common.transport; */ public interface TransportServiceCallback { + TransportServiceCallback EMPTY = new TransportServiceCallback() { + @Override + public void onSuccess(Void msg) { + + } + + @Override + public void onError(Throwable e) { + + } + }; + void onSuccess(T msg); + void onError(Throwable e); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java index ec4f199549..695bb0aa0e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java @@ -31,8 +31,8 @@ public class OAuth2ClientMapperConfig { private String firstNameAttributeKey; private String lastNameAttributeKey; private String tenantNameStrategy; - private String tenantNameStrategyPattern; - private String customerNameStrategyPattern; + private String tenantNamePattern; + private String customerNamePattern; } @Data diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 60d4b1184b..635c7a5dac 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -44,17 +44,19 @@ services: - cassandra tb-core2: env_file: - - tb-node.cassandra.env + - tb-node.hybrid.env depends_on: - kafka - redis + - postgres - cassandra tb-rule-engine1: env_file: - - tb-node.cassandra.env + - tb-node.hybrid.env depends_on: - kafka - redis + - postgres - cassandra tb-rule-engine2: env_file: diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 0b64f43b00..b66073ea44 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,4 +1,4 @@ - +TB_QUEUE_TYPE=kafka REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b290719739..c573274801 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) +queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" js: @@ -25,18 +25,18 @@ kafka: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" - topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" + topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" pubsub: project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" - queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" aws_sqs: access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" region: "TB_QUEUE_AWS_SQS_REGION" - queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" rabbitmq: host: "TB_QUEUE_RABBIT_MQ_HOST" @@ -44,14 +44,14 @@ rabbitmq: virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" username: "TB_QUEUE_RABBIT_MQ_USERNAME" password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" service_bus: namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" - queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" logger: level: "LOGGER_LEVEL" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 3155b051dc..f42b74745f 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "kafka" +queue_type: "kafka" request_topic: "js_eval.requests" js: @@ -25,13 +25,13 @@ kafka: # Kafka Bootstrap Servers servers: "localhost:9092" replication_factor: "1" - topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" pubsub: - queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" + queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" aws_sqs: - queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" + queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" rabbitmq: host: "localhost" @@ -39,10 +39,10 @@ rabbitmq: virtual_host: "/" username: "admin" password: "password" - queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" + queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" service_bus: - queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" + queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/package-lock.json b/msa/js-executor/package-lock.json index 78e7a8eab6..99329281a1 100644 --- a/msa/js-executor/package-lock.json +++ b/msa/js-executor/package-lock.json @@ -4,6 +4,152 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@azure/abort-controller": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.0.1.tgz", + "integrity": "sha512-wP2Jw6uPp8DEDy0n4KNidvwzDjyVV2xnycEIq7nPzj1rHyb/r+t3OPeNT1INZePP2wy5ZqlwyuyOMTi0ePyY1A==", + "requires": { + "tslib": "^1.9.3" + } + }, + "@azure/amqp-common": { + "version": "1.0.0-preview.13", + "resolved": "https://registry.npmjs.org/@azure/amqp-common/-/amqp-common-1.0.0-preview.13.tgz", + "integrity": "sha512-v19NGXFm8Hzr2bj/DSWYc2anaDcoAeFQXJGuBT8QO7eS13vaELQNGaynOGipEcI313A1778R/FFCk4o+dylIiw==", + "requires": { + "@types/async-lock": "^1.1.0", + "@types/is-buffer": "^2.0.0", + "async-lock": "^1.1.3", + "buffer": "^5.2.1", + "debug": "^3.1.0", + "events": "^3.0.0", + "is-buffer": "^2.0.3", + "jssha": "^2.3.1", + "process": "^0.11.10", + "rhea": "^1.0.18", + "rhea-promise": "^0.1.15", + "stream-browserify": "^2.0.2", + "tslib": "^1.9.3", + "url": "^0.11.0", + "util": "^0.11.1" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" + } + } + }, + "@azure/core-auth": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.1.2.tgz", + "integrity": "sha512-IUbP/f3v96dpHgXUwsAjUwDzjlUjawyUhWhGKKB6Qxy+iqppC/pVBPyc6kdpyTe7H30HN+4H3f0lar7Wp9Hx6A==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-tracing": "1.0.0-preview.8", + "@opentelemetry/api": "^0.6.1", + "tslib": "^1.10.0" + } + }, + "@azure/core-http": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-1.1.1.tgz", + "integrity": "sha512-yBxH5CtYaCj0f1CKoi3OjQw5C5Go8TbgNA6Q2rX7XsDpN2eeKu0n3kRvzZnKW+brtO1u3YnBBuBLF2KcGoZv6g==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.1.2", + "@azure/core-tracing": "1.0.0-preview.8", + "@azure/logger": "^1.0.0", + "@opentelemetry/api": "^0.6.1", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.1", + "cross-env": "^6.0.3", + "form-data": "^3.0.0", + "node-fetch": "^2.6.0", + "process": "^0.11.10", + "tough-cookie": "^3.0.1", + "tslib": "^1.10.0", + "tunnel": "^0.0.6", + "uuid": "^3.3.2", + "xml2js": "^0.4.19" + }, + "dependencies": { + "form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "tough-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==", + "requires": { + "ip-regex": "^2.1.0", + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "@azure/core-tracing": { + "version": "1.0.0-preview.8", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.8.tgz", + "integrity": "sha512-ZKUpCd7Dlyfn7bdc+/zC/sf0aRIaNQMDuSj2RhYRFe3p70hVAnYGp3TX4cnG2yoEALp/LTj/XnZGQ8Xzf6Ja/Q==", + "requires": { + "@opencensus/web-types": "0.0.7", + "@opentelemetry/api": "^0.6.1", + "tslib": "^1.10.0" + } + }, + "@azure/logger": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.0.tgz", + "integrity": "sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA==", + "requires": { + "tslib": "^1.9.3" + } + }, + "@azure/service-bus": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@azure/service-bus/-/service-bus-1.1.6.tgz", + "integrity": "sha512-eCJXcJZGWdlVwLEqMcoIqtUrh/NtyFcDDfq/y8gdCOy3Dzuv8JkPTxjdjcxDthwG9mc5Qter3dGOTwh0U8gwiw==", + "requires": { + "@azure/amqp-common": "1.0.0-preview.13", + "@azure/core-http": "^1.0.0", + "@opentelemetry/types": "^0.2.0", + "@types/is-buffer": "^2.0.0", + "@types/long": "^4.0.0", + "buffer": "^5.2.1", + "debug": "^4.1.1", + "is-buffer": "^2.0.3", + "long": "^4.0.0", + "process": "^0.11.10", + "rhea": "^1.0.18", + "rhea-promise": "^0.1.15", + "tslib": "^1.10.0" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "is-buffer": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" + } + } + }, "@babel/parser": { "version": "7.8.4", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.8.4.tgz", @@ -19,6 +165,69 @@ "regenerator-runtime": "^0.13.2" } }, + "@google-cloud/paginator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-2.0.3.tgz", + "integrity": "sha512-kp/pkb2p/p0d8/SKUu4mOq8+HGwF8NPzHWkj+VKrIPQPyMRw8deZtrO/OcSiy9C/7bpfU5Txah5ltUNfPkgEXg==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, + "@google-cloud/precise-date": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-1.0.3.tgz", + "integrity": "sha512-wWnDGh9y3cJHLuVEY8t6un78vizzMWsS7oIWKeFtPj+Ndy+dXvHW0HTx29ZUhen+tswSlQYlwFubvuRP5kKdzQ==" + }, + "@google-cloud/projectify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-1.0.4.tgz", + "integrity": "sha512-ZdzQUN02eRsmTKfBj9FDL0KNDIFNjBn/d6tHQmA/+FImH5DO6ZV8E7FzxMgAUiVAUq41RFAkb25p1oHOZ8psfg==" + }, + "@google-cloud/promisify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz", + "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ==" + }, + "@google-cloud/pubsub": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@google-cloud/pubsub/-/pubsub-1.7.2.tgz", + "integrity": "sha512-/TziioDSV4FS4wKF1sIaQ+1gvE+um83oHz1nRsZ3L87uWSoOciBjJAcocgPjqrpnW441+Nuw4w0QdSUV1Lka/g==", + "requires": { + "@google-cloud/paginator": "^2.0.0", + "@google-cloud/precise-date": "^1.0.0", + "@google-cloud/projectify": "^1.0.0", + "@google-cloud/promisify": "^1.0.0", + "@types/duplexify": "^3.6.0", + "@types/long": "^4.0.0", + "arrify": "^2.0.0", + "async-each": "^1.0.1", + "extend": "^3.0.2", + "google-auth-library": "^5.5.0", + "google-gax": "^1.14.2", + "is-stream-ended": "^0.1.4", + "lodash.snakecase": "^4.1.1", + "p-defer": "^3.0.0", + "protobufjs": "^6.8.1" + } + }, + "@grpc/grpc-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.0.3.tgz", + "integrity": "sha512-JKV3f5Bv2TZxK6eJSB9EarsZrnLxrvcFNwI9goq0YRXa3S6NNoCSnI3cG3lkXVIJ03Wng1WXe76kc2JQtRe7AQ==", + "requires": { + "semver": "^6.2.0" + } + }, + "@grpc/proto-loader": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.4.tgz", + "integrity": "sha512-HTM4QpI9B2XFkPz7pjwMyMgZchJ93TVkL3kWPW8GDMDKYxsMnmf4w2TNMJK7+KNiYHS5cJrCEAFlF+AwtXWVPA==", + "requires": { + "lodash.camelcase": "^4.3.0", + "protobufjs": "^6.8.6" + } + }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -45,23 +254,193 @@ "fastq": "^1.6.0" } }, + "@opencensus/web-types": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/@opencensus/web-types/-/web-types-0.0.7.tgz", + "integrity": "sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g==" + }, + "@opentelemetry/api": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-0.6.1.tgz", + "integrity": "sha512-wpufGZa7tTxw7eAsjXJtiyIQ42IWQdX9iUQp7ACJcKo1hCtuhLU+K2Nv1U6oRwT1oAlZTE6m4CgWKZBhOiau3Q==", + "requires": { + "@opentelemetry/context-base": "^0.6.1" + } + }, + "@opentelemetry/context-base": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-base/-/context-base-0.6.1.tgz", + "integrity": "sha512-5bHhlTBBq82ti3qPT15TRxkYTFPPQWbnkkQkmHPtqiS1XcTB69cEKd3Jm7Cfi/vkPoyxapmePE9tyA7EzLt8SQ==" + }, + "@opentelemetry/types": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/types/-/types-0.2.0.tgz", + "integrity": "sha512-GtwNB6BNDdsIPAYEdpp3JnOGO/3AJxjPvny53s3HERBdXSJTGQw8IRhiaTEX0b3w9P8+FwFZde4k+qkjn67aVw==" + }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "@types/async-lock": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/async-lock/-/async-lock-1.1.2.tgz", + "integrity": "sha512-j9n4bb6RhgFIydBe0+kpjnBPYumDaDyU8zvbWykyVMkku+c2CSu31MZkLeaBfqIwU+XCxlDpYDfyMQRkM0AkeQ==" + }, "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", "dev": true }, + "@types/duplexify": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", + "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", + "requires": { + "@types/node": "*" + } + }, + "@types/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==", + "requires": { + "@types/node": "*" + } + }, + "@types/is-buffer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/is-buffer/-/is-buffer-2.0.0.tgz", + "integrity": "sha512-0f7N/e3BAz32qDYvgB4d2cqv1DqUwvGxHkXsrucICn8la1Vb6Yl6Eg8mPScGwUiqHJeE7diXlzaK+QMA9m4Gxw==", + "requires": { + "@types/node": "*" + } + }, + "@types/long": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", + "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" + }, + "@types/node": { + "version": "13.13.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.4.tgz", + "integrity": "sha512-x26ur3dSXgv5AwKS0lNfbjpCakGIduWU1DU91Zz58ONRWrIKGunmZBNv4P7N+e27sJkiGDsw/3fT4AtsqQBrBA==" + }, + "@types/node-fetch": { + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz", + "integrity": "sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==", + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + }, + "dependencies": { + "form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, + "@types/tunnel": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.1.tgz", + "integrity": "sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A==", + "requires": { + "@types/node": "*" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, + "agent-base": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz", + "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==", + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "ajv": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz", "integrity": "sha512-nCprB/0syFYy9fVYU1ox1l2KN8S9I+tziH8D4zdZuLT3N6RMlGSGt5FSTpAiHB/Whv8Qs1cWHma1aMKZyaHRKA==", - "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -69,6 +448,47 @@ "uri-js": "^4.2.2" } }, + "amqplib": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.5.tgz", + "integrity": "sha512-sWx1hbfHbyKMw6bXOK2k6+lHL8TESWxjAx5hG8fBtT7wcxoXNIsFxZMnFyBjxt3yL14vn7WqBDe5U6BGOadtLg==", + "requires": { + "bitsyntax": "~0.1.0", + "bluebird": "^3.5.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "safe-buffer": "~5.1.2", + "url-parse": "~1.4.3" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -152,11 +572,15 @@ "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", "dev": true }, + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dev": true, "requires": { "safer-buffer": "~2.1.0" } @@ -164,8 +588,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "dev": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assign-symbols": { "version": "1.0.0", @@ -184,14 +607,17 @@ "async-each": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", - "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", - "dev": true + "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==" + }, + "async-lock": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.2.2.tgz", + "integrity": "sha512-uczz62z2fMWOFbyo6rG4NlV2SdxugJT6sZA2QcfB1XaSjEiOh8CuOb/TttyMnYQCda6nkWecJe465tGQDPJiKw==" }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", - "dev": true + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "atob": { "version": "2.1.2", @@ -199,17 +625,126 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, + "aws-sdk": { + "version": "2.669.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.669.0.tgz", + "integrity": "sha512-kuVcSRpDzvkgmeSmMX6Q32eTOb8UeihhUdavMrvUOP6fzSU19cNWS9HAIkYOi/jrEDK85cCZxXjxqE3JGZIGcw==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "dev": true + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", - "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==", - "dev": true + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" + }, + "azure-common": { + "version": "0.9.22", + "resolved": "https://registry.npmjs.org/azure-common/-/azure-common-0.9.22.tgz", + "integrity": "sha512-0r9tK9D+1xl2/VPVtfmGmtkMqfooiBLS87fX+Ab0hOCPVVe/6CgVC4in0wSf2Ta8r65DbvxV5P4/t8fp8Q3EsQ==", + "requires": { + "dateformat": "1.0.2-1.2.3", + "duplexer": "~0.1.1", + "envconf": "~0.0.4", + "request": "^2.81.0", + "through": "~2.3.4", + "tunnel": "~0.0.2", + "underscore": "1.4.x", + "validator": "^9.4.1", + "xml2js": "^0.4.19", + "xmlbuilder": "0.4.3" + }, + "dependencies": { + "underscore": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz", + "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ=" + }, + "xmlbuilder": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.3.tgz", + "integrity": "sha1-xGFLp04K0ZbmCcknLNnh3bKKilg=" + } + } + }, + "azure-sb": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/azure-sb/-/azure-sb-0.11.1.tgz", + "integrity": "sha512-ZYgPeSDMD99i/Em+6wT78zvBkJ/dbh2ypb4DbqQ1Flaif5vWJFzC/iKxxcq/vq+THWoO3+UbqWa0JNXnW3zAvw==", + "requires": { + "azure-common": "^0.9.22", + "mpns": "2.1.3", + "underscore": "^1.8.3", + "wns": "~0.5.3" + } }, "balanced-match": { "version": "1.0.0", @@ -272,15 +807,24 @@ } } }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dev": true, "requires": { "tweetnacl": "^0.14.3" } }, + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + }, "binary-extensions": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", @@ -297,6 +841,41 @@ "file-uri-to-path": "1.0.0" } }, + "bitsyntax": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.1.0.tgz", + "integrity": "sha512-ikAdCnrloKmFOugAfxWws89/fPc+nw0OOG1IzIE72uSOg/A3cYptKCjSUhDTuj7fhsJtzkzlv7l3b8PzRHLN0Q==", + "requires": { + "buffer-more-ints": "~1.0.0", + "debug": "~2.6.9", + "safe-buffer": "~5.1.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -351,6 +930,25 @@ } } }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, + "buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, "byline": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", @@ -389,8 +987,7 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", - "dev": true + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "chalk": { "version": "2.4.2", @@ -522,7 +1119,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -581,6 +1177,52 @@ "capture-stack-trace": "^1.0.0" } }, + "cross-env": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", + "integrity": "sha512-+KqxF6LCvfhWvADcDPqo64yVIB31gv/jQulX2NGzKS/g3GEVz6/pt4wjHFtFWsHMddebWD/sDthJemzM4MaAag==", + "requires": { + "cross-spawn": "^7.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.2.tgz", + "integrity": "sha512-PD6G8QG3S4FK/XCGFbEQrDqO2AnMMsy0meR7lerlIOHAAbkuavGU/pOqprrlvfTNjvowivTeBsjebAL0NSoMxw==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", @@ -607,16 +1249,19 @@ "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } }, + "dateformat": { + "version": "1.0.2-1.2.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.2-1.2.3.tgz", + "integrity": "sha1-sCIMAt6YYXQztyhRz0fePfLNvuk=" + }, "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "dev": true, "requires": { "ms": "^2.1.1" } @@ -683,8 +1328,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", - "dev": true + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "diagnostics": { "version": "1.1.1", @@ -714,22 +1358,74 @@ "is-obj": "^1.0.0" } }, - "duplexer3": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", - "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", - "dev": true - }, + "duplexer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=" + }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true + }, + "duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "dev": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "enabled": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", @@ -738,11 +1434,24 @@ "env-variable": "0.0.x" } }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "env-variable": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", "integrity": "sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==" }, + "envconf": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/envconf/-/envconf-0.0.4.tgz", + "integrity": "sha1-hWda+6I3xD+Y3i1GrcDlMqTc9Is=" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -788,6 +1497,16 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, + "events": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz", + "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg==" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -862,8 +1581,7 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "dev": true + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "extend-shallow": { "version": "3.0.2", @@ -954,14 +1672,12 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "dev": true + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", - "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", - "dev": true + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==" }, "fast-glob": { "version": "3.1.1", @@ -1033,8 +1749,7 @@ "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-levenshtein": { "version": "2.0.6", @@ -1047,6 +1762,11 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" }, + "fast-text-encoding": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.2.tgz", + "integrity": "sha512-5rQdinSsycpzvAoHga2EDn+LRX1d5xLFsuNG0Kg61JrAT/tASXcLL0nf/33v+sAxlQcfYmWbTURa1mmAf55jGw==" + }, "fastq": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.6.0.tgz", @@ -1108,14 +1828,12 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "dev": true + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -1226,12 +1944,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1251,7 +1971,8 @@ "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", @@ -1399,6 +2120,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1723,6 +2445,34 @@ } } }, + "gaxios": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.4.tgz", + "integrity": "sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + } + } + }, + "gcp-metadata": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz", + "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==", + "requires": { + "gaxios": "^2.1.0", + "json-bigint": "^0.3.0" + } + }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", @@ -1739,7 +2489,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -1788,6 +2537,67 @@ "slash": "^3.0.0" } }, + "google-auth-library": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz", + "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.1.0", + "gcp-metadata": "^3.4.0", + "gtoken": "^4.1.0", + "jws": "^4.0.0", + "lru-cache": "^5.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + } + } + }, + "google-gax": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.15.3.tgz", + "integrity": "sha512-3JKJCRumNm3x2EksUTw4P1Rad43FTpqrtW9jzpf3xSMYXx+ogaqTM1vGo7VixHB4xkAyATXVIa3OcNSh8H9zsQ==", + "requires": { + "@grpc/grpc-js": "~1.0.3", + "@grpc/proto-loader": "^0.5.1", + "@types/fs-extra": "^8.0.1", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^3.6.0", + "google-auth-library": "^5.0.0", + "is-stream-ended": "^0.1.4", + "lodash.at": "^4.6.0", + "lodash.has": "^4.5.2", + "node-fetch": "^2.6.0", + "protobufjs": "^6.8.9", + "retry-request": "^4.0.0", + "semver": "^6.0.0", + "walkdir": "^0.4.0" + } + }, + "google-p12-pem": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz", + "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==", + "requires": { + "node-forge": "^0.9.0" + } + }, "got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -1813,17 +2623,26 @@ "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", "dev": true }, + "gtoken": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz", + "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==", + "requires": { + "gaxios": "^2.1.0", + "google-p12-pem": "^2.0.0", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "dev": true + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "dev": true, "requires": { "ajv": "^6.5.5", "har-schema": "^2.0.0" @@ -1871,13 +2690,36 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", "sshpk": "^1.7.0" } }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", @@ -1923,6 +2765,11 @@ "p-is-promise": "^3.0.0" } }, + "ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=" + }, "is-accessor-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", @@ -2115,11 +2962,15 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, + "is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==" + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "dev": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "is-windows": { "version": "1.0.2", @@ -2135,8 +2986,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isobject": { "version": "3.0.1", @@ -2147,8 +2997,12 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", - "dev": true + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, "js-yaml": { "version": "3.13.1", @@ -2162,26 +3016,30 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "dev": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "json-bigint": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "requires": { + "bignumber.js": "^7.0.0" + } }, "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", - "dev": true + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", - "dev": true + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "json5": { "version": "1.0.1", @@ -2204,7 +3062,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -2212,6 +3069,30 @@ "verror": "1.10.0" } }, + "jssha": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/jssha/-/jssha-2.4.2.tgz", + "integrity": "sha512-/jsi/9C0S70zfkT/4UlKQa5E1xKurDnXcQizcww9JSR/Fv+uIbWM2btG+bFcL3iNoK9jIGS0ls9HWLr1iw0kFg==" + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "kafkajs": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.12.0.tgz", @@ -2258,6 +3139,26 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, + "lodash.at": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=" + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" + }, + "lodash.has": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" + }, + "lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha1-OdcUo1NXFHg3rv1ktdy7Fr7Nj40=" + }, "logform": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz", @@ -2342,17 +3243,20 @@ "to-regex": "^3.0.2" } }, + "mime": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.5.tgz", + "integrity": "sha512-3hQhEUF027BuxZjQA3s7rIv/7VCQPa27hN9u9g87sEkWaKwQPuXOkVKtOeiyUrnWqTDiOs8Ed2rwg733mB0R5w==" + }, "mime-db": { "version": "1.43.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", - "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==", - "dev": true + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" }, "mime-types": { "version": "2.1.26", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", - "dev": true, "requires": { "mime-db": "1.43.0" } @@ -2414,6 +3318,11 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, + "mpns": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/mpns/-/mpns-2.1.3.tgz", + "integrity": "sha512-gPLNoVqwYoKUmNYZ2shMSdaE2XvHSRxWNzyG4DUi6Av7MSujyeOw/nj61nnQeuV/vke5E0Dni468xn0qxTHIZQ==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2487,6 +3396,16 @@ "to-regex": "^3.0.1" } }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, + "node-forge": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" + }, "nodemon": { "version": "1.19.4", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.4.tgz", @@ -2540,8 +3459,7 @@ "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "dev": true + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "object-copy": { "version": "0.1.0", @@ -2597,6 +3515,14 @@ "isobject": "^3.0.1" } }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, "one-time": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", @@ -2622,6 +3548,11 @@ "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, + "p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==" + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -2699,8 +3630,7 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", - "dev": true + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "picomatch": { "version": "2.2.1", @@ -2898,6 +3828,11 @@ "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", "dev": true }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -2909,6 +3844,26 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "protobufjs": { + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.9.0.tgz", + "integrity": "sha512-LlGVfEWDXoI/STstRDdZZKb/qusoAWUnmLg9R8OLSO473mBLWHowx8clbX5/+mKDEI+v7GzjoK9tRPZMMcoTrg==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -2918,8 +3873,7 @@ "psl": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", - "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==", - "dev": true + "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==" }, "pstree.remy": { "version": "1.1.7", @@ -2930,14 +3884,22 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", - "dev": true + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "querystringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==" }, "rc": { "version": "1.2.8", @@ -3061,7 +4023,6 @@ "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", - "dev": true, "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -3094,6 +4055,11 @@ "throttleit": "^1.0.0" } }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, "resolve": { "version": "1.15.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz", @@ -3115,12 +4081,49 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, + "retry-request": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", + "requires": { + "debug": "^4.1.1", + "through2": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true }, + "rhea": { + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/rhea/-/rhea-1.0.20.tgz", + "integrity": "sha512-qj4LSEykJ0SEYESQLg9Vee6VXH5xHN1pYj7ozPeUk+l+S1OaGKx1FugAu+g+3pPwK46WXV1PJD9XiRx8+tS4cw==", + "requires": { + "debug": "0.8.0 - 3.5.0" + } + }, + "rhea-promise": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/rhea-promise/-/rhea-promise-0.1.15.tgz", + "integrity": "sha512-+6uilZXSJGyiqVeHQI3Krv6NTAd8cWRCY2uyCxmzR4/5IFtBqqFem1HV2OiwSj0Gu7OFChIJDfH2JyjN7J0vRA==", + "requires": { + "debug": "^3.1.0", + "rhea": "^1.0.4", + "tslib": "^1.9.3" + } + }, "run-parallel": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", @@ -3144,8 +4147,12 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "semver": { "version": "6.3.0", @@ -3392,7 +4399,6 @@ "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "dev": true, "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -3431,6 +4437,44 @@ } } }, + "stream-browserify": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "requires": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "stream-meter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", @@ -3472,6 +4516,11 @@ } } }, + "stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -3540,6 +4589,19 @@ "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=", "dev": true }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "requires": { + "readable-stream": "2 || 3" + } + }, "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", @@ -3601,7 +4663,6 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dev": true, "requires": { "psl": "^1.1.28", "punycode": "^2.1.1" @@ -3612,11 +4673,20 @@ "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, + "tslib": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", + "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==" + }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, "requires": { "safe-buffer": "^5.0.1" } @@ -3624,8 +4694,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "dev": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-check": { "version": "0.3.2", @@ -3668,6 +4737,11 @@ } } }, + "underscore": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz", + "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==" + }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", @@ -3780,7 +4854,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "dev": true, "requires": { "punycode": "^2.1.0" } @@ -3791,6 +4864,31 @@ "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=", "dev": true }, + "url": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "url-parse-lax": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", @@ -3806,6 +4904,21 @@ "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", "dev": true }, + "util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -3814,25 +4927,38 @@ "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "dev": true + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, "uuid-parse": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz", "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==" }, + "uuid-random": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/uuid-random/-/uuid-random-1.3.0.tgz", + "integrity": "sha512-FSIlv8RFRPOjcHeDYStV7u6aJRfp+THrcWkbAJpw51JCyQLDxsFz+4dHgTYP8hSpZeSMXBpb/1qrK4bodXpSRA==" + }, + "validator": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==" + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } }, + "walkdir": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", + "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -3942,12 +5068,22 @@ } } }, + "wns": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/wns/-/wns-0.5.4.tgz", + "integrity": "sha512-WYiJ7khIwUGBD5KAm+YYmwJDDRzFRs4YGAjtbFSoRIdbn9Jcix3p9khJmpvBTXGommaKkvduAn+pc9l4d9yzVQ==" + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, "write-file-atomic": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", @@ -3965,6 +5101,20 @@ "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=", "dev": true }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 5870ab8efa..f8ec68f1e2 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -22,6 +22,7 @@ "azure-sb": "^0.11.1", "long": "^4.0.0", "uuid-parse": "^1.0.0", + "uuid-random": "^1.3.0", "winston": "^3.0.0", "winston-daily-rotate-file": "^3.2.1" }, diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index 0396824af1..a74d9d5b57 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -19,6 +19,7 @@ const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), logger = require('../config/logger')._logger('awsSqsTemplate'); +const uuid = require('uuid-random'); const requestTopic = config.get('request_topic'); @@ -26,10 +27,10 @@ const accessKeyId = config.get('aws_sqs.access_key_id'); const secretAccessKey = config.get('aws_sqs.secret_access_key'); const region = config.get('aws_sqs.region'); const AWS = require('aws-sdk'); -const queueProperties = config.get('aws_sqs.queue-properties'); -const poolInterval = config.get('js.response_poll_interval'); +const queueProperties = config.get('aws_sqs.queue_properties'); +const pollInterval = config.get('js.response_poll_interval'); -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; +let queueAttributes = {FifoQueue: 'true'}; let sqsClient; let requestQueueURL; const queueUrls = new Map(); @@ -51,7 +52,12 @@ function AwsSqsProducer() { queueUrls.set(responseTopic, responseQueueUrl); } - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; + let params = { + MessageBody: msgBody, + QueueUrl: responseQueueUrl, + MessageGroupId: 'js_eval', + MessageDeduplicationId: uuid() + }; return new Promise((resolve, reject) => { sqsClient.sendMessage(params, function (err, data) { @@ -74,11 +80,13 @@ function AwsSqsProducer() { const queues = await getQueues(); - queues.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - queueUrls.set(queueName, queueUrl); - }) + if (queues) { + queues.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + queueUrls.set(queueName, queueUrl); + }); + } parseQueueProperties(); @@ -95,6 +103,7 @@ function AwsSqsProducer() { WaitTimeSeconds: poolInterval / 1000 }; while (!stopped) { + let pollStartTs = new Date().getTime(); const messages = await new Promise((resolve, reject) => { sqsClient.receiveMessage(params, function (err, data) { if (err) { @@ -127,6 +136,11 @@ function AwsSqsProducer() { //do nothing } }); + } else { + let pollDuration = new Date().getTime() - pollStartTs; + if (pollDuration < pollInterval) { + await sleep(pollInterval - pollDuration); + } } } } catch (e) { @@ -175,6 +189,12 @@ function parseQueueProperties() { }); } +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + process.on('exit', () => { stopped = true; logger.info('Aws Sqs client stopped.'); diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 6a09338427..33637cee81 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -20,7 +20,7 @@ const config = require('config'), logger = require('../config/logger')._logger('kafkaTemplate'), KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; const replicationFactor = config.get('kafka.replication_factor'); -const topicProperties = config.get('kafka.topic-properties'); +const topicProperties = config.get('kafka.topic_properties'); let kafkaClient; let kafkaAdmin; diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 17e1b56e1d..8cb264de64 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -24,7 +24,7 @@ const {PubSub} = require('@google-cloud/pubsub'); const projectId = config.get('pubsub.project_id'); const credentials = JSON.parse(config.get('pubsub.service_account')); const requestTopic = config.get('request_topic'); -const queueProperties = config.get('pubsub.queue-properties'); +const queueProperties = config.get('pubsub.queue_properties'); let pubSubClient; @@ -98,23 +98,32 @@ function PubSubProducer() { async function createTopic(topic) { if (!topics.includes(topic)) { - await pubSubClient.createTopic(topic); + try { + await pubSubClient.createTopic(topic); + logger.info('Created new Pub/Sub topic: %s', topic); + } catch (e) { + logger.info('Pub/Sub topic already exists'); + } topics.push(topic); - logger.info('Created new Pub/Sub topic: %s', topic); } await createSubscription(topic) } async function createSubscription(topic) { if (!subscriptions.includes(topic)) { - await pubSubClient.createSubscription(topic, topic, { - topic: topic, - subscription: topic, - ackDeadlineSeconds: queueProps['ackDeadlineInSec'], - messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} - }); + try { + await pubSubClient.createSubscription(topic, topic, { + topic: topic, + subscription: topic, + ackDeadlineSeconds: queueProps['ackDeadlineInSec'], + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} + }); + logger.info('Created new Pub/Sub subscription: %s', topic); + } catch (e) { + logger.info('Pub/Sub subscription already exists.'); + } + subscriptions.push(topic); - logger.info('Created new Pub/Sub subscription: %s', topic); } } diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js index 1a2905c3a0..2aeb5c02db 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -26,23 +26,23 @@ const port = config.get('rabbitmq.port'); const vhost = config.get('rabbitmq.virtual_host'); const username = config.get('rabbitmq.username'); const password = config.get('rabbitmq.password'); -const queueProperties = config.get('rabbitmq.queue-properties'); -const poolInterval = config.get('js.response_poll_interval'); +const queueProperties = config.get('rabbitmq.queue_properties'); +const pollInterval = config.get('js.response_poll_interval'); const amqp = require('amqplib/callback_api'); -let queueParams = {durable: false, exclusive: false, autoDelete: false}; +let queueOptions = {durable: false, exclusive: false, autoDelete: false}; let connection; let channel; let stopped = false; -const responseTopics = []; +let queues = []; function RabbitMqProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { - if (!responseTopics.includes(responseTopic)) { + if (!queues.includes(responseTopic)) { await createQueue(responseTopic); - responseTopics.push(responseTopic); + queues.push(responseTopic); } let data = JSON.stringify( @@ -98,6 +98,7 @@ function RabbitMqProducer() { const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer()); while (!stopped) { + let pollStartTs = new Date().getTime(); let message = await new Promise((resolve, reject) => { channel.get(requestTopic, {}, function (err, msg) { if (err) { @@ -112,7 +113,10 @@ function RabbitMqProducer() { messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); channel.ack(message); } else { - await sleep(poolInterval); + let pollDuration = new Date().getTime() - pollStartTs; + if (pollDuration < pollInterval) { + await sleep(pollInterval - pollDuration); + } } } } catch (e) { @@ -123,16 +127,18 @@ function RabbitMqProducer() { })(); function parseQueueProperties() { + let args = {}; const props = queueProperties.split(';'); props.forEach(p => { const delimiterPosition = p.indexOf(':'); - queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + args[p.substring(0, delimiterPosition)] = +p.substring(delimiterPosition + 1); }); + queueOptions['arguments'] = args; } -function createQueue(topic) { +async function createQueue(topic) { return new Promise((resolve, reject) => { - channel.assertQueue(topic, queueParams, function (err) { + channel.assertQueue(topic, queueOptions, function (err) { if (err) { reject(err); } else { diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js index 034921afb7..99316930e3 100644 --- a/msa/js-executor/queue/serviceBusTemplate.js +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -26,7 +26,7 @@ const requestTopic = config.get('request_topic'); const namespaceName = config.get('service_bus.namespace_name'); const sasKeyName = config.get('service_bus.sas_key_name'); const sasKey = config.get('service_bus.sas_key'); -const queueProperties = config.get('service_bus.queue-properties'); +const queueProperties = config.get('service_bus.queue_properties'); let sbClient; let receiverClient; @@ -140,6 +140,7 @@ function parseQueueProperties() { properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); }); queueOptions = { + DuplicateDetection: 'false', MaxSizeInMegabytes: properties['maxSizeInMb'], DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, LockDuration: `PT${properties['lockDurationInSec']}S` diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 58361016c4..e57ba62c11 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -16,7 +16,7 @@ const config = require('config'), logger = require('./config/logger')._logger('main'); -const serviceType = config.get('service-type'); +const serviceType = config.get('queue_type'); switch (serviceType) { case 'kafka': logger.info('Starting kafka template.'); diff --git a/pom.xml b/pom.xml index 8844edc9e3..e70ae211cd 100755 --- a/pom.xml +++ b/pom.xml @@ -30,10 +30,10 @@ ${basedir} thingsboard - 2.2.4.RELEASE + 2.2.6.RELEASE 2.1.2.RELEASE - 5.2.2.RELEASE - 5.2.2.RELEASE + 5.2.6.RELEASE + 5.2.3.RELEASE 2.2.4.RELEASE 3.1.0 0.7.0 @@ -62,14 +62,14 @@ 2.6.2 1.7 2.0 - 1.4.3 + 1.6.2 4.2.0 3.5.5 3.11.4 1.22.1 1.16.18 1.1.0 - 4.1.45.Final + 4.1.49.Final 1.5.0 4.8.0 2.19.1 @@ -96,7 +96,7 @@ 1.25 1.3.10 1.11.747 - 1.84.0 + 1.105.0 3.2.0 1.5.0 1.4.3 @@ -471,12 +471,12 @@ org.springframework.security spring-security-oauth2-client - ${spring.version} + ${spring-security.version} org.springframework.security spring-security-oauth2-jose - ${spring.version} + ${spring-security.version} org.springframework.boot @@ -591,8 +591,8 @@ ${rabbitmq.version} - javax.mail - mail + com.sun.mail + javax.mail ${mail.version} @@ -610,6 +610,12 @@ org.apache.zookeeper zookeeper ${zookeeper.version} + + + log4j + log4j + + com.jayway.jsonpath @@ -692,6 +698,12 @@ com.github.fge json-schema-validator ${json-schema-validator.version} + + + javax.mail + mailapi + + com.typesafe.akka @@ -928,12 +940,6 @@ com.microsoft.azure azure-servicebus ${azure-servicebus.version} - - - com.microsoft.azure - adal4j - - org.passay diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 8070bac591..58104ea07e 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -93,5 +93,10 @@ spring-data-redis provided + + com.sun.mail + javax.mail + provided + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index 63fb59bed5..b0413c6dd7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -81,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode) { let index = 0; datasources.forEach((datasource) => { diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index f41be60581..fa7cb2c3c8 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -15,11 +15,12 @@ /// import _ from 'lodash'; -import { Observable, Observer, of, Subject } from 'rxjs'; -import { finalize, map, share } from 'rxjs/operators'; +import { Observable, Subject } from 'rxjs'; +import { finalize, share } from 'rxjs/operators'; import base64js from 'base64-js'; import { Datasource } from '@app/shared/models/widget.models'; -import { FormattedData } from '@app/modules/home/components/widget/lib/maps/map-models'; + +const varsRegex = /\${([^}]*)}/g; export function onParentScrollOrWindowResize(el: Node): Observable { const scrollSubject = new Subject(); @@ -33,7 +34,7 @@ export function onParentScrollOrWindowResize(el: Node): Observable { scrollParentNode.addEventListener('scroll', eventListenerObject); }); window.addEventListener('resize', eventListenerObject); - const shared = scrollSubject.pipe( + return scrollSubject.pipe( finalize(() => { scrollParentNodes.forEach((scrollParentNode) => { scrollParentNode.removeEventListener('scroll', eventListenerObject); @@ -42,18 +43,13 @@ export function onParentScrollOrWindowResize(el: Node): Observable { }), share() ); - return shared; } export function isLocalUrl(url: string): boolean { const parser = document.createElement('a'); parser.href = url; const host = parser.hostname; - if (host === 'localhost' || host === '127.0.0.1') { - return true; - } else { - return false; - } + return host === 'localhost' || host === '127.0.0.1'; } export function animatedScroll(element: HTMLElement, scrollTop: number, delay?: number) { @@ -68,8 +64,7 @@ export function animatedScroll(element: HTMLElement, scrollTop: number, delay?: element.scrollTop = to; } else { currentTime += increment; - const val = easeInOut(currentTime, start, remaining, duration); - element.scrollTop = val; + element.scrollTop = easeInOut(currentTime, start, remaining, duration); if (currentTime < duration) { setTimeout(animateScroll, increment); } @@ -110,6 +105,15 @@ export function isString(value: any): boolean { return typeof value === 'string'; } +export function isEmpty(obj: any): boolean { + for (const key of Object.keys(obj)) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + return false; + } + } + return true; +} + export function formatValue(value: any, dec?: number, units?: string, showZeroDecimals?: boolean): string | undefined { if (isDefined(value) && value !== null && isNumeric(value)) { @@ -118,7 +122,7 @@ export function formatValue(value: any, dec?: number, units?: string, showZeroDe formatted = formatted.toFixed(dec); } if (!showZeroDecimals) { - formatted = (Number(formatted) * 1); + formatted = (Number(formatted)); } formatted = formatted.toString(); if (isDefined(units) && units.length > 0) { @@ -154,15 +158,13 @@ export function deleteNullProperties(obj: any) { export function objToBase64(obj: any): string { const json = JSON.stringify(obj); const encoded = utf8Encode(json); - const b64Encoded: string = base64js.fromByteArray(encoded); - return b64Encoded; + return base64js.fromByteArray(encoded); } export function base64toObj(b64Encoded: string): any { const encoded: Uint8Array | number[] = base64js.toByteArray(b64Encoded); const json = utf8Decode(encoded); - const obj = JSON.parse(json); - return obj; + return JSON.parse(json); } function utf8Encode(str: string): Uint8Array | number[] { @@ -223,11 +225,13 @@ function scrollParents(node: Node): Node[] { return scrollParentNodes; } -function hashCode(str) { +export function hashCode(str: string): number { let hash = 0; - let i; - let char; - if (str.length === 0) return hash; + let i: number; + let char: number; + if (str.length === 0) { + return hash; + } for (i = 0; i < str.length; i++) { char = str.charCodeAt(i); // tslint:disable-next-line:no-bitwise @@ -238,6 +242,15 @@ function hashCode(str) { return hash; } +export function objectHashCode(obj: any): number { + let hash = 0; + if (obj) { + const str = JSON.stringify(obj); + hash = hashCode(str); + } + return hash; +} + function easeInOut( currentTime: number, startTime: number, @@ -433,29 +446,21 @@ export function getDescendantProp(obj: any, path: string): any { return path.split('.').reduce((acc, part) => acc && acc[part], obj); } -export function imageLoader(imageUrl: string): Observable { - return new Observable((observer: Observer) => { - const image = new Image(); - image.style.position = 'absolute'; - image.style.left = '-99999px'; - image.style.top = '-99999px'; - image.onload = () => { - observer.next(image); - document.body.removeChild(image); - observer.complete(); - }; - image.onerror = err => { - observer.error(err); - document.body.removeChild(image); - observer.complete(); - }; - document.body.appendChild(image) - image.src = imageUrl; - }); +export function insertVariable(pattern: string, name: string, value: any): string { + let result = pattern; + let match = varsRegex.exec(pattern); + while (match !== null) { + const variable = match[0]; + const variableName = match[1]; + if (variableName === name) { + result = result.split(variable).join(value); + } + match = varsRegex.exec(pattern); + } + return result; } export function createLabelFromDatasource(datasource: Datasource, pattern: string) { - const varsRegex = /\$\{([^}]*)\}/g; let label = pattern; if (!datasource) { return label; @@ -482,142 +487,6 @@ export function createLabelFromDatasource(datasource: Datasource, pattern: strin return label; } -const imageAspectMap = {}; - -export function aspectCache(imageUrl: string): Observable { - if (imageUrl?.length) { - const hash = hashCode(imageUrl); - let aspect = imageAspectMap[hash]; - if (aspect) { - return of(aspect); - } - else return imageLoader(imageUrl).pipe(map(image => { - aspect = image.width / image.height; - imageAspectMap[hash] = aspect; - return aspect; - })) - } -} - -export function parseArray(input: any[]): any[] { - return _(input).groupBy(el => el?.datasource?.entityName) - .values().value().map((entityArray, dsIndex) => - entityArray[0].data.map((el, i) => { - const obj = { - entityName: entityArray[0]?.datasource?.entityName, - $datasource: entityArray[0]?.datasource, - dsIndex, - time: el[0], - deviceType: null - }; - entityArray.filter(el => el.data.length).forEach(entity => { - obj[entity?.dataKey?.label] = entity?.data[i][1]; - obj[entity?.dataKey?.label + '|ts'] = entity?.data[0][0]; - if (entity?.dataKey?.label === 'type') { - obj.deviceType = entity?.data[0][1]; - } - }); - return obj; - }) - ); -} - -export function parseData(input: any[]): FormattedData[] { - return _(input).groupBy(el => el?.datasource?.entityName) - .values().value().map((entityArray, i) => { - const obj = { - entityName: entityArray[0]?.datasource?.entityName, - $datasource: entityArray[0]?.datasource as Datasource, - dsIndex: i, - deviceType: null - }; - entityArray.filter(el => el.data.length).forEach(el => { - obj[el?.dataKey?.label] = el?.data[0][1]; - obj[el?.dataKey?.label + '|ts'] = el?.data[0][0]; - if (el?.dataKey?.label === 'type') { - obj.deviceType = el?.data[0][1]; - } - }); - return obj; - }); -} - -export function safeExecute(func: Function, params = []) { - let res = null; - if (func && typeof (func) === 'function') { - try { - res = func(...params); - } - catch (err) { - console.log('error in external function:', err); - res = null; - } - } - return res; -} - -export function parseFunction(source: any, params: string[] = ['def']): Function { - let res = null; - if (source?.length) { - try { - res = new Function(...params, source); - } - catch (err) { - res = null; - } - } - return res; -} - -export function parseTemplate(template: string, data: { $datasource?: Datasource, [key: string]: any }, - translateFn?: (key: string) => string) { - let res = ''; - try { - if (template.match(//g, 'a>') - .replace(/name=(\'|")(.*?)(\'|")/g, `class='tb-custom-action' id='$2'`); - } - if (translateFn) { - template = translateFn(template); - } - template = createLabelFromDatasource(data.$datasource, template); - const formatted = template.match(/\$\{([^}]*)\:\d*\}/g); - if (formatted) - formatted.forEach(value => { - const [variable, digits] = value.replace('${', '').replace('}', '').split(':'); - data[variable] = padValue(data[variable], +digits); - if (data[variable] === 'NaN') data[variable] = ''; - template = template.replace(value, '${' + variable + '}'); - }); - const variables = template.match(/\$\{.*?\}/g); - if (variables) { - variables.forEach(variable => { - variable = variable.replace('${', '').replace('}', ''); - if (!data[variable]) - data[variable] = ''; - }) - } - const compiled = _.template(template); - res = compiled(data); - } - catch (ex) { - console.log(ex, template) - } - return res; -} - -export let parseWithTranslation = { - translate(): string { - throw console.error('Translate not assigned'); - }, - parseTemplate(template: string, data: object, forceTranslate = false): string { - return parseTemplate(forceTranslate ? this.translate(template) : template, data, this?.translate); - }, - setTranslate(translateFn: (key: string, defaultTranslation?: string) => string) { - this.translate = translateFn; - } -} - export function padValue(val: any, dec: number): string { let strVal; let n; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 1eb7b39ca9..58f7b73aec 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -33,7 +33,7 @@ import { Datasource, WidgetActionDescriptor, WidgetConfig } from '@shared/models import { IWidgetSubscription } from '@core/api/widget-api.models'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { deepClone, isDefined, isNumber, createLabelFromDatasource } from '@core/utils'; +import { deepClone, isDefined, isNumber, createLabelFromDatasource, hashCode } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; @@ -304,7 +304,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, const cssString = constructTableCssString(this.widgetConfig); const cssParser = new cssjs(); cssParser.testMode = false; - const namespace = 'alarms-table-' + this.utils.hashCode(cssString); + const namespace = 'alarms-table-' + hashCode(cssString); cssParser.cssPreviewNamespace = namespace; cssParser.createStyleElement(namespace, cssString); $(this.elementRef.nativeElement).addClass(namespace); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts index 9056c40f19..1b8881e8a0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts @@ -37,7 +37,7 @@ import { import { BaseData } from '@shared/models/base-data'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; -import { deepClone } from '@core/utils'; +import { deepClone, hashCode } from '@core/utils'; import { defaultNodeIconFunction, defaultNodeOpenedFunction, @@ -186,7 +186,7 @@ export class EntitiesHierarchyWidgetComponent extends PageComponent implements O const cssString = constructTableCssString(this.widgetConfig); const cssParser = new cssjs(); cssParser.testMode = false; - const namespace = 'entities-hierarchy-' + this.utils.hashCode(cssString); + const namespace = 'entities-hierarchy-' + hashCode(cssString); cssParser.cssPreviewNamespace = namespace; cssParser.createStyleElement(namespace, cssString); $(this.elementRef.nativeElement).addClass(namespace); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 54f8344890..b50274bba8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -39,7 +39,7 @@ import { import { IWidgetSubscription } from '@core/api/widget-api.models'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { deepClone, isDefined, isNumber, createLabelFromDatasource } from '@core/utils'; +import { deepClone, isDefined, isNumber, createLabelFromDatasource, hashCode } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; @@ -226,7 +226,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni const cssString = constructTableCssString(this.widgetConfig); const cssParser = new cssjs(); cssParser.testMode = false; - const namespace = 'entities-table-' + this.utils.hashCode(cssString); + const namespace = 'entities-table-' + hashCode(cssString); cssParser.cssPreviewNamespace = namespace; cssParser.createStyleElement(namespace, cssString); $(this.elementRef.nativeElement).addClass(namespace); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts index 64a8621920..93477c1782 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet-map.ts @@ -14,19 +14,26 @@ /// limitations under the License. /// -import L, { LatLngTuple, LatLngBounds, Point, MarkerClusterGroupOptions, markerClusterGroup } from 'leaflet'; +import L, { LatLngBounds, LatLngTuple, markerClusterGroup, MarkerClusterGroupOptions } from 'leaflet'; import 'leaflet-providers'; import 'leaflet.markercluster/dist/leaflet.markercluster'; -import { MapSettings, MarkerSettings, FormattedData, UnitedMapSettings, PolygonSettings, PolylineSettings } from './map-models'; +import { + FormattedData, + MapSettings, + MarkerSettings, + PolygonSettings, + PolylineSettings, + UnitedMapSettings +} from './map-models'; import { Marker } from './markers'; import { BehaviorSubject, Observable } from 'rxjs'; import { filter } from 'rxjs/operators'; import { Polyline } from './polyline'; import { Polygon } from './polygon'; import { DatasourceData } from '@app/shared/models/widget.models'; -import { safeExecute } from '@app/core/utils'; +import { safeExecute } from '@home/components/widget/lib/maps/maps-utils'; export default abstract class LeafletMap { @@ -38,13 +45,11 @@ export default abstract class LeafletMap { map$: BehaviorSubject = new BehaviorSubject(null); ready$: Observable = this.map$.pipe(filter(map => !!map)); options: UnitedMapSettings; - isMarketCluster: boolean; bounds: L.LatLngBounds; - newMarker: L.Marker; datasources: FormattedData[]; markersCluster; - constructor(public $container: HTMLElement, options: UnitedMapSettings) { + protected constructor(public $container: HTMLElement, options: UnitedMapSettings) { this.options = options; } @@ -125,7 +130,7 @@ export default abstract class LeafletMap { addMarker.setPosition('topright') } L.Control.AddMarker = L.Control.extend({ - onAdd(map) { + onAdd() { const img = L.DomUtil.create('img') as any; img.src = `assets/add_location.svg`; img.style.width = '32px'; @@ -138,7 +143,7 @@ export default abstract class LeafletMap { draggableImg.on('dragend', dragListener) return img; }, - onRemove(map) { + onRemove() { }, dragMarker: this.dragMarker } as any); @@ -166,7 +171,7 @@ export default abstract class LeafletMap { this.datasources = dataSources; } - public saveMarkerLocation(e) { + public saveMarkerLocation(_e) { } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts index c3219f90c2..523f078bc7 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { LatLngTuple, LeafletMouseEvent } from 'leaflet'; +import { LatLngTuple } from 'leaflet'; import { Datasource } from '@app/shared/models/widget.models'; export type GenericFunction = (data: FormattedData, dsData: FormattedData[], dsIndex: number) => string; @@ -116,7 +116,7 @@ export type PolygonSettings = { autocloseTooltip: boolean; tooltipFunction: GenericFunction; showTooltipAction: string; - tooltipAction: object; + tooltipAction: { [name: string]: actionsHandler }; tooltipPattern: string; useTooltipFunction: boolean; polygonClick: { [name: string]: actionsHandler }; @@ -152,6 +152,6 @@ export interface HistorySelectSettings { buttonColor: string; } -export type actionsHandler = ($event: Event | LeafletMouseEvent, datasource: Datasource) => void; +export type actionsHandler = ($event: Event, datasource: Datasource) => void; export type UnitedMapSettings = MapSettings & PolygonSettings & MarkerSettings & PolylineSettings; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts index 330e739c31..3f4c4ab3f4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-widget2.ts @@ -31,11 +31,10 @@ import { } from './schemes'; import { MapWidgetStaticInterface, MapWidgetInterface } from './map-widget.interface'; import { OpenStreetMap, TencentMap, GoogleMap, HEREMap, ImageMap } from './providers'; -import { parseFunction, parseArray, parseData, parseWithTranslation } from '@core/utils'; import { initSchema, addToSchema, mergeSchemes, addCondition, addGroupInfo } from '@core/schema-utils'; import { of, Subject } from 'rxjs'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; -import { getDefCenterPosition } from './maps-utils'; +import { getDefCenterPosition, parseArray, parseData, parseFunction, parseWithTranslation } from './maps-utils'; import { JsonSettingsSchema, WidgetActionDescriptor, DatasourceType, widgetType, Datasource } from '@shared/models/widget.models'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope, DataKeyType, LatestTelemetry } from '@shared/models/telemetry/telemetry.models'; @@ -138,7 +137,7 @@ export class MapWidgetController implements MapWidgetInterface { else return ''; } - getDescriptors(name: string): { [name: string]: ($event: Event) => void } { + getDescriptors(name: string): { [name: string]: ($event: Event, datasource: Datasource) => void } { const descriptors = this.ctx.actionsApi.getActionDescriptors(name); const actions = {}; descriptors.forEach(descriptor => { @@ -150,9 +149,10 @@ export class MapWidgetController implements MapWidgetInterface { onInit() { } - private onCustomAction(descriptor: WidgetActionDescriptor, $event: any, entityInfo: Datasource) { - if ($event && $event.stopPropagation) { - $event?.stopPropagation(); + private onCustomAction(descriptor: WidgetActionDescriptor, $event: Event, entityInfo: Datasource) { + if ($event) { + $event.preventDefault(); + $event.stopPropagation(); } const { entityId, entityName, entityLabel, entityType } = entityInfo; this.ctx.actionsApi.handleWidgetAction($event, descriptor, { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts index 0cb3f8765a..ae056ad01c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/maps-utils.ts @@ -15,8 +15,12 @@ /// import L from 'leaflet'; -import { MarkerSettings, PolygonSettings, PolylineSettings } from './map-models'; +import { FormattedData, MarkerSettings, PolygonSettings, PolylineSettings } from './map-models'; import { Datasource } from '@app/shared/models/widget.models'; +import _ from 'lodash'; +import { Observable, Observer, of } from 'rxjs'; +import { map } from 'rxjs/operators'; +import { createLabelFromDatasource, hashCode, padValue } from '@core/utils'; export function createTooltip(target: L.Layer, settings: MarkerSettings | PolylineSettings | PolygonSettings, @@ -49,7 +53,7 @@ export function createTooltip(target: L.Layer, export function getRatio(firsMoment: number, secondMoment: number, intermediateMoment: number): number { return (intermediateMoment - firsMoment) / (secondMoment - firsMoment); -}; +} export function findAngle(startPoint, endPoint) { let angle = -Math.atan2(endPoint.latitude - startPoint.latitude, endPoint.longitude - startPoint.longitude); @@ -65,3 +69,170 @@ export function getDefCenterPosition(position) { return position; return [0, 0]; } + + +const imageAspectMap = {}; + +function imageLoader(imageUrl: string): Observable { + return new Observable((observer: Observer) => { + const image = new Image(); + image.style.position = 'absolute'; + image.style.left = '-99999px'; + image.style.top = '-99999px'; + image.onload = () => { + observer.next(image); + document.body.removeChild(image); + observer.complete(); + }; + image.onerror = err => { + observer.error(err); + document.body.removeChild(image); + observer.complete(); + }; + document.body.appendChild(image) + image.src = imageUrl; + }); +} + +export function aspectCache(imageUrl: string): Observable { + if (imageUrl?.length) { + const hash = hashCode(imageUrl); + let aspect = imageAspectMap[hash]; + if (aspect) { + return of(aspect); + } + else return imageLoader(imageUrl).pipe(map(image => { + aspect = image.width / image.height; + imageAspectMap[hash] = aspect; + return aspect; + })) + } +} + +function parseTemplate(template: string, data: { $datasource?: Datasource, [key: string]: any }, + translateFn?: (key: string) => string) { + let res = ''; + try { + if (template.match(//g, 'a>') + .replace(/name=(['"])(.*?)(['"])/g, `class='tb-custom-action' id='$2'`); + } + if (translateFn) { + template = translateFn(template); + } + template = createLabelFromDatasource(data.$datasource, template); + const formatted = template.match(/\${([^}]*):\d*}/g); + if (formatted) + formatted.forEach(value => { + const [variable, digits] = value.replace('${', '').replace('}', '').split(':'); + data[variable] = padValue(data[variable], +digits); + if (data[variable] === 'NaN') data[variable] = ''; + template = template.replace(value, '${' + variable + '}'); + }); + const variables = template.match(/\${.*?}/g); + if (variables) { + variables.forEach(variable => { + variable = variable.replace('${', '').replace('}', ''); + if (!data[variable]) + data[variable] = ''; + }) + } + const compiled = _.template(template); + res = compiled(data); + } + catch (ex) { + console.log(ex, template) + } + return res; +} + +type TranslateFunc = (key: string, defaultTranslation?: string) => string; + +export const parseWithTranslation = { + + translateFn: null, + + translate(key: string, defaultTranslation?: string): string { + if (this.translateFn) { + return this.translateFn(key, defaultTranslation); + } else { + throw console.error('Translate not assigned'); + } + }, + parseTemplate(template: string, data: object, forceTranslate = false): string { + return parseTemplate(forceTranslate ? this.translate(template) : template, data, this.translate.bind(this)); + }, + setTranslate(translateFn: TranslateFunc) { + this.translateFn = translateFn; + } +} + +export function parseData(input: any[]): FormattedData[] { + return _(input).groupBy(el => el?.datasource?.entityName) + .values().value().map((entityArray, i) => { + const obj = { + entityName: entityArray[0]?.datasource?.entityName, + $datasource: entityArray[0]?.datasource as Datasource, + dsIndex: i, + deviceType: null + }; + entityArray.filter(el => el.data.length).forEach(el => { + obj[el?.dataKey?.label] = el?.data[0][1]; + obj[el?.dataKey?.label + '|ts'] = el?.data[0][0]; + if (el?.dataKey?.label === 'type') { + obj.deviceType = el?.data[0][1]; + } + }); + return obj; + }); +} + +export function parseArray(input: any[]): any[] { + return _(input).groupBy(el => el?.datasource?.entityName) + .values().value().map((entityArray, dsIndex) => + entityArray[0].data.map((el, i) => { + const obj = { + entityName: entityArray[0]?.datasource?.entityName, + $datasource: entityArray[0]?.datasource, + dsIndex, + time: el[0], + deviceType: null + }; + entityArray.filter(e => e.data.length).forEach(entity => { + obj[entity?.dataKey?.label] = entity?.data[i][1]; + obj[entity?.dataKey?.label + '|ts'] = entity?.data[0][0]; + if (entity?.dataKey?.label === 'type') { + obj.deviceType = entity?.data[0][1]; + } + }); + return obj; + }) + ); +} + +export function parseFunction(source: any, params: string[] = ['def']): (...args: any[]) => any { + let res = null; + if (source?.length) { + try { + res = new Function(...params, source); + } + catch (err) { + res = null; + } + } + return res; +} + +export function safeExecute(func: (...args: any[]) => any, params = []) { + let res = null; + if (func && typeof (func) === 'function') { + try { + res = func(...params); + } + catch (err) { + console.log('error in external function:', err); + res = null; + } + } + return res; +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts index cc2da88514..388f08eda5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/markers.ts @@ -14,10 +14,9 @@ /// limitations under the License. /// -import { aspectCache, parseWithTranslation, safeExecute } from '@app/core/utils'; import L, { LeafletMouseEvent } from 'leaflet'; import { FormattedData, MarkerSettings } from './map-models'; -import { createTooltip } from './maps-utils'; +import { aspectCache, createTooltip, parseWithTranslation, safeExecute } from './maps-utils'; import tinycolor from 'tinycolor2'; export class Marker { @@ -50,7 +49,7 @@ export class Marker { this.leafletMarker.on('click', (event: LeafletMouseEvent) => { for (const action in this.settings.markerClick) { if (typeof (this.settings.markerClick[action]) === 'function') { - this.settings.markerClick[action](event, this.data.$datasource); + this.settings.markerClick[action](event.originalEvent, this.data.$datasource); } } }); @@ -184,4 +183,4 @@ export class Marker { setMarkerPosition(latLng) { this.leafletMarker.setLatLng(latLng); } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/polygon.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/polygon.ts index 1a817c31d9..4f1fc9dd72 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/polygon.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/polygon.ts @@ -15,10 +15,9 @@ /// import L, { LatLngExpression, LatLngTuple, LeafletMouseEvent } from 'leaflet'; -import { createTooltip } from './maps-utils'; -import { PolygonSettings, FormattedData } from './map-models'; +import { createTooltip, parseWithTranslation, safeExecute } from './maps-utils'; +import { PolygonSettings } from './map-models'; import { DatasourceData } from '@app/shared/models/widget.models'; -import { safeExecute, parseWithTranslation } from '@app/core/utils'; export class Polygon { @@ -46,7 +45,7 @@ export class Polygon { this.leafletPoly.on('click', (event: LeafletMouseEvent) => { for (const action in this.settings.polygonClick) { if (typeof (this.settings.polygonClick[action]) === 'function') { - this.settings.polygonClick[action](event, polyData.datasource); + this.settings.polygonClick[action](event.originalEvent, polyData.datasource); } } }); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/polyline.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/polyline.ts index 9f20d3ff68..ba001d7c1e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/polyline.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/polyline.ts @@ -17,8 +17,8 @@ import L, { PolylineDecoratorOptions } from 'leaflet'; import 'leaflet-polylinedecorator'; -import { safeExecute } from '@app/core/utils'; import { PolylineSettings } from './map-models'; +import { safeExecute } from '@home/components/widget/lib/maps/maps-utils'; export class Polyline { @@ -95,4 +95,4 @@ export class Polyline { setPolylineLatLngs(latLngs) { this.leafletPoly.setLatLngs(latLngs); } -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts index f5263bb4a3..b62262bf53 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/google-map.ts @@ -29,7 +29,7 @@ export class GoogleMap extends LeafletMap { super($container, options); this.loadGoogle(() => { const map = L.map($container).setView(options?.defaultCenterPosition, options?.defaultZoomLevel); - const roads = (L.gridLayer as any).googleMutant({ + (L.gridLayer as any).googleMutant({ type: options?.gmDefaultMapType || 'roadmap' }).addTo(map); super.setMap(map); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts index ab9dc60ef4..bd12185ad6 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts @@ -17,9 +17,9 @@ import L, { LatLngLiteral, LatLngBounds, LatLngTuple } from 'leaflet'; import LeafletMap from '../leaflet-map'; import { UnitedMapSettings } from '../map-models'; -import { aspectCache, parseFunction } from '@app/core/utils'; import { Observable } from 'rxjs'; import { map, filter, switchMap } from 'rxjs/operators'; +import { aspectCache, parseFunction } from '@home/components/widget/lib/maps/maps-utils'; const maxZoom = 4;// ? diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts index 5e55af55f3..5008758e69 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts @@ -558,32 +558,6 @@ export const markerClusteringSettingsSchema = ] }; -export const markerClusteringSettingsSchemaGoogle = -{ - schema: { - title: 'Marker Clustering Configuration Google', - type: 'object', - properties: { - gridSize: { - title: 'Maximum radius that a cluster will cover in pixels', - type: 'number', - default: 60 - }, - minimumClusterSize: { - title: 'The minimum number of markers in a cluster', - type: 'number' - } - }, - required: [] - }, - form: [ - 'gridSize', - 'minimumClusterSize' - ] -}; - - - export const markerClusteringSettingsSchemaLeaflet = { schema: { @@ -1271,4 +1245,4 @@ export const tripAnimationSchema = { } ] }] -} \ No newline at end of file +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 592cb4d286..5523ca2f03 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -40,7 +40,7 @@ import { } from '@shared/models/widget.models'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { isDefined, isNumber } from '@core/utils'; +import { hashCode, isDefined, isNumber } from '@core/utils'; import cssjs from '@core/css/css'; import { PageLink } from '@shared/models/page/page-link'; import { Direction, SortOrder, sortOrderFromString } from '@shared/models/page/sort-order'; @@ -230,7 +230,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI const cssParser = new cssjs(); cssParser.testMode = false; - const namespace = 'ts-table-' + this.utils.hashCode(cssString); + const namespace = 'ts-table-' + hashCode(cssString); cssParser.cssPreviewNamespace = namespace; cssParser.createStyleElement(namespace, cssString); $(this.elementRef.nativeElement).addClass(namespace); diff --git a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts index 57ab874bae..47b7e7f743 100644 --- a/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/trip-animation/trip-animation.component.ts @@ -22,12 +22,11 @@ import { interpolateOnPointSegment } from 'leaflet-geometryutil'; import { AfterViewInit, ChangeDetectorRef, Component, Input, OnInit, SecurityContext, ViewChild } from '@angular/core'; import { MapWidgetController, TbMapWidgetV2 } from '../lib/maps/map-widget2'; import { MapProviders } from '../lib/maps/map-models'; -import { parseArray, parseWithTranslation, safeExecute, parseTemplate } from '@app/core/utils'; import { initSchema, addToSchema, addGroupInfo } from '@app/core/schema-utils'; import { tripAnimationSchema } from '../lib/maps/schemes'; import { DomSanitizer } from '@angular/platform-browser'; import { WidgetContext } from '@app/modules/home/models/widget-component.models'; -import { findAngle, getRatio } from '../lib/maps/maps-utils'; +import { findAngle, getRatio, parseArray, parseWithTranslation, safeExecute } from '../lib/maps/maps-utils'; import { JsonSettingsSchema, WidgetConfig } from '@shared/models/widget.models'; import moment from 'moment'; diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html index 1e6c21cb7c..a73278c36e 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/manage-dashboard-customers-dialog.component.html @@ -34,7 +34,7 @@ {{ labelText | translate }} diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.ts b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.ts index ddbbec4f21..07d22b5262 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/states/entity-state-controller.component.ts @@ -23,7 +23,7 @@ import { StateControllerComponent } from './state-controller.component'; import { StatesControllerService } from '@home/pages/dashboard/states/states-controller.service'; import { EntityId } from '@app/shared/models/id/entity-id'; import { UtilsService } from '@core/services/utils.service'; -import { base64toObj, objToBase64 } from '@app/core/utils'; +import { base64toObj, insertVariable, isEmpty, objToBase64 } from '@app/core/utils'; import { DashboardUtilsService } from '@core/services/dashboard-utils.service'; import { EntityService } from '@core/http/entity.service'; import { EntityType } from '@shared/models/entity-type.models'; @@ -123,11 +123,10 @@ export class EntityStateControllerComponent extends StateControllerComponent imp if (this.states && this.states[id]) { this.resolveEntity(params).subscribe( () => { - const newState: StateObject = { + this.stateObject[this.stateObject.length - 1] = { id, params }; - this.stateObject[this.stateObject.length - 1] = newState; this.gotoState(this.stateObject[this.stateObject.length - 1].id, true, openRightLayout); } ); @@ -203,14 +202,14 @@ export class EntityStateControllerComponent extends StateControllerComponent imp const targetParams = params && params.targetEntityParamName ? params[params.targetEntityParamName] : params; const entityName = targetParams && targetParams.entityName ? targetParams.entityName : ''; const entityLabel = targetParams && targetParams.entityLabel ? targetParams.entityLabel : ''; - result = this.utils.insertVariable(stateName, 'entityName', entityName); - result = this.utils.insertVariable(result, 'entityLabel', entityLabel); + result = insertVariable(stateName, 'entityName', entityName); + result = insertVariable(result, 'entityLabel', entityLabel); for (const prop of Object.keys(params)) { if (params[prop] && params[prop].entityName) { - result = this.utils.insertVariable(result, prop + ':entityName', params[prop].entityName); + result = insertVariable(result, prop + ':entityName', params[prop].entityName); } if (params[prop] && params[prop].entityLabel) { - result = this.utils.insertVariable(result, prop + ':entityLabel', params[prop].entityLabel); + result = insertVariable(result, prop + ':entityLabel', params[prop].entityLabel); } } } @@ -276,20 +275,13 @@ export class EntityStateControllerComponent extends StateControllerComponent imp if (this.stateObject.length === 1) { const state = this.stateObject[0]; const rootStateId = this.dashboardUtils.getRootStateId(this.states); - if (state.id === rootStateId && (!state.params || this.isEmpty(state.params))) { + if (state.id === rootStateId && (!state.params || isEmpty(state.params))) { return true; } } return false; } - private isEmpty(obj: any): boolean { - for (const key of Object.keys(obj)) { - return !Object.prototype.hasOwnProperty.call(obj, key); - } - return true; - } - private resolveEntity(params: StateParams): Observable { if (params && params.targetEntityParamName) { params = params[params.targetEntityParamName]; @@ -313,10 +305,7 @@ export class EntityStateControllerComponent extends StateControllerComponent imp } private isEntityResolved(params: StateParams): boolean { - if (!params.entityName || !params.entityName.length) { - return false; - } - return true; + return !(!params.entityName || !params.entityName.length); } private getStateObjById(id: string): StateObject { diff --git a/ui-ngx/src/app/shared/pipe/public-api.ts b/ui-ngx/src/app/shared/pipe/public-api.ts index 72edfae70d..4f801b7d51 100644 --- a/ui-ngx/src/app/shared/pipe/public-api.ts +++ b/ui-ngx/src/app/shared/pipe/public-api.ts @@ -20,4 +20,3 @@ export * from './keyboard-shortcut.pipe'; export * from './milliseconds-to-time-string.pipe'; export * from './nospace.pipe'; export * from './truncate.pipe'; -export * from './template.pipe'; diff --git a/ui-ngx/src/app/shared/pipe/template.pipe.ts b/ui-ngx/src/app/shared/pipe/template.pipe.ts deleted file mode 100644 index 2cce658adb..0000000000 --- a/ui-ngx/src/app/shared/pipe/template.pipe.ts +++ /dev/null @@ -1,25 +0,0 @@ -/// -/// Copyright © 2016-2020 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Pipe, PipeTransform } from '@angular/core'; -import { parseTemplate, parseWithTranslation } from '@app/core/utils'; - -@Pipe({ name: 'tbParseTemplate' }) -export class TbTemplatePipe implements PipeTransform { - transform(template, data): string { - return parseWithTranslation.parseTemplate(template, data); - } -} diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 4914873095..e55a39632b 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -131,7 +131,6 @@ import { LedLightComponent } from '@shared/components/led-light.component'; import { TbJsonToStringDirective } from '@shared/components/directives/tb-json-to-string.directive'; import { JsonObjectEditDialogComponent } from '@shared/components/dialog/json-object-edit-dialog.component'; import { HistorySelectorComponent } from './components/time/history-selector/history-selector.component'; -import { TbTemplatePipe } from '@shared/pipe/template.pipe'; import { EntityGatewaySelectComponent } from '@shared/components/entity/entity-gateway-select.component'; import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list.component'; @@ -215,7 +214,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list HighlightPipe, TruncatePipe, TbJsonPipe, - TbTemplatePipe, KeyboardShortcutPipe, TbJsonToStringDirective, JsonObjectEditDialogComponent, @@ -377,7 +375,6 @@ import { QueueTypeListComponent } from '@shared/components/queue/queue-type-list HighlightPipe, TruncatePipe, TbJsonPipe, - TbTemplatePipe, KeyboardShortcutPipe, TranslateModule, JsonObjectEditDialogComponent, diff --git a/ui/src/app/locale/locale.constant-de_DE.json b/ui/src/app/locale/locale.constant-de_DE.json index ef5c6c9344..1f8b3b6fbe 100644 --- a/ui/src/app/locale/locale.constant-de_DE.json +++ b/ui/src/app/locale/locale.constant-de_DE.json @@ -1149,7 +1149,7 @@ "total": "Gesamt" }, "login": { - "login": "Login", + "login": "Anmelden", "request-password-reset": "Passwortzurücksetzung anfordern", "reset-password": "Passwort zurücksetzen", "create-password": "Passwort erstellen", @@ -1163,7 +1163,9 @@ "new-password": "Neues Passwort", "new-password-again": "Neues Passwort wiederholen", "password-link-sent-message": "Der Link zum Zurücksetzen des Passworts wurde erfolgreich versendet!", - "email": "E-Mail" + "email": "E-Mail", + "login-with": "Mit {{name}} anmelden", + "or": "oder" }, "position": { "top": "Oben", diff --git a/ui/src/app/locale/locale.constant-el_GR.json b/ui/src/app/locale/locale.constant-el_GR.json index 8d4951f28f..bf51056f94 100644 --- a/ui/src/app/locale/locale.constant-el_GR.json +++ b/ui/src/app/locale/locale.constant-el_GR.json @@ -1705,7 +1705,9 @@ "password-link-sent-message": "Ο σύνδεσμος επαναφοράς κωδικού πρόσβασης στάλθηκε με επιτυχία!", "email": "Email", "no-account": "Δεν έχετε λογαριασμό;", - "create-account": "Δημιουργία λογαριασμού" + "create-account": "Δημιουργία λογαριασμού", + "login-with": "Σύνδεση μέσω {{name}}", + "or": "ή" }, "signup": { "firstname": "Όνομα", diff --git a/ui/src/app/locale/locale.constant-es_ES.json b/ui/src/app/locale/locale.constant-es_ES.json index 8394e9ed65..df5752f54b 100644 --- a/ui/src/app/locale/locale.constant-es_ES.json +++ b/ui/src/app/locale/locale.constant-es_ES.json @@ -1229,7 +1229,9 @@ "new-password": "Nueva contraseña", "new-password-again": "Repita la nueva contraseña", "password-link-sent-message": "¡El enlace para el restablecer la contraseña fue enviado correctamente!", - "email": "Correo electrónico" + "email": "Correo electrónico", + "login-with": "Iniciar sesión con {{name}}", + "or": "o" }, "position": { "top": "Superior", diff --git a/ui/src/app/locale/locale.constant-fr_FR.json b/ui/src/app/locale/locale.constant-fr_FR.json index c9285ffb37..274063445f 100644 --- a/ui/src/app/locale/locale.constant-fr_FR.json +++ b/ui/src/app/locale/locale.constant-fr_FR.json @@ -1197,7 +1197,7 @@ "create-password": "Créer un mot de passe", "email": "Email", "forgot-password": "Mot de passe oublié?", - "login": "Login", + "login": "Connexion", "new-password": "Nouveau mot de passe", "new-password-again": "nouveau mot de passe", "password-again": "Mot de passe à nouveau", @@ -1208,7 +1208,9 @@ "request-password-reset": "Demander la réinitialisation du mot de passe", "reset-password": "Réinitialiser le mot de passe", "sign-in": "Veuillez vous connecter", - "username": "Nom d'utilisateur (courriel)" + "username": "Nom d'utilisateur (courriel)", + "login-with": "Se connecter avec {{name}}", + "or": "ou" }, "position": { "bottom": "Bas", diff --git a/ui/src/app/locale/locale.constant-it_IT.json b/ui/src/app/locale/locale.constant-it_IT.json index 1f694eabc7..8d246c361d 100644 --- a/ui/src/app/locale/locale.constant-it_IT.json +++ b/ui/src/app/locale/locale.constant-it_IT.json @@ -1160,7 +1160,7 @@ "total": "totale" }, "login": { - "login": "Login", + "login": "Accedi", "request-password-reset": "Richiesta reset password", "reset-password": "Reset Password", "create-password": "Crea Password", @@ -1174,7 +1174,9 @@ "new-password": "Nuova password", "new-password-again": "Ripeti nuova password", "password-link-sent-message": "Link reset password inviato con successo!", - "email": "Email" + "email": "Email", + "login-with": "Accedi con {{name}}", + "or": "o" }, "position": { "top": "Alto", diff --git a/ui/src/app/locale/locale.constant-ja_JA.json b/ui/src/app/locale/locale.constant-ja_JA.json index f40122e2d6..ec372ce513 100644 --- a/ui/src/app/locale/locale.constant-ja_JA.json +++ b/ui/src/app/locale/locale.constant-ja_JA.json @@ -1024,7 +1024,9 @@ "new-password": "新しいパスワード", "new-password-again": "新しいパスワードを再入力", "password-link-sent-message": "パスワードリセットリンクが正常に送信されました！", - "email": "Eメール" + "email": "Eメール", + "login-with": "{{name}}でログイン", + "or": "または" }, "position": { "top": "上", diff --git a/ui/src/app/locale/locale.constant-ko_KR.json b/ui/src/app/locale/locale.constant-ko_KR.json index 1760334552..e5f09184b9 100644 --- a/ui/src/app/locale/locale.constant-ko_KR.json +++ b/ui/src/app/locale/locale.constant-ko_KR.json @@ -934,7 +934,9 @@ "new-password": "새 비밀번호", "new-password-again": "새 비밀번호 확인", "password-link-sent-message": "비밀번호 재설정 링크가 성공적으로 전송되었습니다!", - "email": "이메일" + "email": "이메일", + "login-with": "{{name}}으로 로그인", + "or": "또는" }, "position": { "top": "상단", diff --git a/ui/src/app/locale/locale.constant-ro_RO.json b/ui/src/app/locale/locale.constant-ro_RO.json index 612bec7d7e..0ec2c81794 100644 --- a/ui/src/app/locale/locale.constant-ro_RO.json +++ b/ui/src/app/locale/locale.constant-ro_RO.json @@ -1230,7 +1230,7 @@ } }, "login": { - "login": "Intră în Cont", + "login": "Conectare", "request-password-reset": "Solicită Resetarea Parolei", "reset-password": "Resetează Parolă", "create-password": "Creează Parolă", @@ -1245,7 +1245,9 @@ "new-password": "Parolă nouă", "new-password-again": "Verificare parolă nouă", "password-link-sent-message": "Ți-am trimis pe eMail un link pentru resetarea parolei", - "email": "eMail" + "email": "eMail", + "login-with": "Conectare cu {{name}}", + "or": "sau" }, "position": { "top": "Sus", diff --git a/ui/src/app/locale/locale.constant-tr_TR.json b/ui/src/app/locale/locale.constant-tr_TR.json index 46e484ef99..3f4441f79e 100644 --- a/ui/src/app/locale/locale.constant-tr_TR.json +++ b/ui/src/app/locale/locale.constant-tr_TR.json @@ -1090,7 +1090,7 @@ "total": "toplam" }, "login": { - "login": "Oturum aç", + "login": "Giriş Yap", "request-password-reset": "Parola Sıfırlama İsteği Gönder", "reset-password": "Parola Sıfırla", "create-password": "Parola Oluştur", @@ -1104,7 +1104,9 @@ "new-password": "Yeni parola", "new-password-again": "Yeni parola tekrarı", "password-link-sent-message": "Parola sıfırlama e-postası başarıyla gönderildi!", - "email": "E-posta" + "email": "E-posta", + "login-with": "{{name}} ile Giriş Yap", + "or": "ya da" }, "position": { "top": "Üst",