diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7f028932f2..5a0e291ac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -106,6 +107,14 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired + @Getter + private TbApiUsageStateService apiUsageStateService; + + @Autowired + @Getter + private TbApiUsageClient apiUsageClient; + @Autowired @Getter @Setter diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 826596db56..fea5558540 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -40,8 +40,6 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; @@ -150,15 +148,12 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { - if (msg.getEntityId().getEntityType() == EntityType.TENANT_PROFILE) { - tenantProfileCache.evict(new TenantProfileId(msg.getEntityId().getId())); - } else { + if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) { log.warn("Message has system tenant id: {}", msg); } } else { - if (msg.getEntityId().getEntityType() == EntityType.TENANT) { + if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) { TenantId tenantId = new TenantId(msg.getEntityId().getId()); - tenantProfileCache.evict(tenantId); if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); deletedTenants.add(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 3ab4ac8b9c..6749ed69ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import java.util.Collections; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -312,7 +314,7 @@ class DefaultTbContext implements TbContext { @Override public ScriptEngine createJsScriptEngine(String script, String... argNames) { - return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); + return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); } @Override @@ -479,8 +481,16 @@ class DefaultTbContext implements TbContext { } @Override - public void addProfileListener(Consumer listener) { - mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), listener); + public void removeRuleNodeStateForEntity(EntityId entityId) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId); + } + mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId); + } + + @Override + public void addDeviceProfileListeners(Consumer profileListener, BiConsumer deviceListener) { + mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 5ca394be35..ad5f97ab48 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.queue.TbClusterService; import java.util.ArrayList; @@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor> nodeRoutes; private final RuleChainService service; private final TbClusterService clusterService; + private final TbApiUsageClient apiUsageClient; private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext - , TbActorRef parent, TbActorRef self) { + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) { super(systemContext, tenantId, ruleChain.getId()); + this.apiUsageClient = systemContext.getApiUsageClient(); this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index eb46d7c098..0f828dfeb4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } } + protected void destroyRuleChains() { + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + ctx.stop(new TbEntityActorId(ruleChain.getId())); + } + } + protected void visit(RuleChain entity, TbActorRef actorRef) { if (entity != null && entity.isRoot()) { rootChain = entity; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index db55ff8edf..95d21090bd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; /** * @author Andrew Shvayka @@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor extends Abstract this.entityId = id; } + protected TenantProfileConfiguration getTenantProfileConfiguration() { + return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration(); + } + public abstract String getComponentName(); public abstract void start(TbActorCtx context) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 161239be7e..72351a480a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -57,6 +58,7 @@ public class TenantActor extends RuleChainManagerActor { private boolean isRuleEngineForCurrentTenant; private boolean isCore; + private ApiUsageState apiUsageState; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -74,19 +76,24 @@ public class TenantActor extends RuleChainManagerActor { cantFindTenant = true; log.info("[{}] Started tenant actor for missing tenant.", tenantId); } else { + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenant.getId())); + // This Service may be started for specific tenant only. Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngineForCurrentTenant) { try { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { - log.info("[{}] Going to init rule chains", tenantId); - initRuleChains(); + if (apiUsageState.isReExecEnabled()) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } } else { isRuleEngineForCurrentTenant = false; } @@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor { } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); -// TODO: throw this in 3.1? -// throw new TbActorException("Failed to init actor", e); } } @@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor { if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; queueMsg.getTbMsg().getCallback().onSuccess(); - } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ + } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) { TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; transportMsg.getCallback().onSuccess(); } @@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor { return; } TbMsg tbMsg = msg.getTbMsg(); - if (tbMsg.getRuleChainId() == null) { - if (getRootChainActor() != null) { - getRootChainActor().tell(msg); + if (apiUsageState.isReExecEnabled()) { + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); - log.info("[{}] No Root Chain: {}", tenantId, msg); + try { + ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); + } catch (TbActorNotRegisteredException ex) { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } else { - try { - ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); - } catch (TbActorNotRegisteredException ex) { - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); - //TODO: 3.1 Log it to dead letters queue; - tbMsg.getCallback().onSuccess(); - } + log.trace("[{}] Ack message because Rule Engine is disabled", tenantId); + tbMsg.getCallback().onSuccess(); } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - getOrCreateActor(msg.getRuleChainId()).tell(msg); + if (apiUsageState.isReExecEnabled()) { + getOrCreateActor(msg.getRuleChainId()).tell(msg); + } } private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { @@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor { } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { + if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { + ApiUsageState old = apiUsageState; + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId)); + if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId); + destroyRuleChains(); + } else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId); + initRuleChains(); + } + } if (isRuleEngineForCurrentTenant) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index c8fde63c4d..03ccc9fb93 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; @@ -119,6 +120,8 @@ public class DeviceController extends BaseController { tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(), + device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 6d80c05c05..7392f74231 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -160,7 +160,7 @@ public class RuleChainController extends BaseController { public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { try { checkNotNull(request); - checkNotNull(request.getName()); + checkParameter(request.getName(), "name"); RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); @@ -347,7 +347,7 @@ public class RuleChainController extends BaseController { String errorText = ""; ScriptEngine engine = null; try { - engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); + engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames); TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 83e3d98b1b..a42cbb0f91 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -93,6 +93,8 @@ public class TenantController extends BaseController { } tenantProfileCache.evict(tenant.getId()); tbClusterService.onTenantChange(tenant, null); + tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(), + newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); return tenant; } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java new file mode 100644 index 0000000000..f2dfab54d4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.Getter; + +public enum ApiFeature { + TRANSPORT("transportApiState"), DB("dbApiState"), RE("ruleEngineApiState"), JS("jsExecutionApiState"); + + @Getter + private final String apiStateKey; + + ApiFeature(String apiStateKey) { + this.apiStateKey = apiStateKey; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java new file mode 100644 index 0000000000..2c8879ba94 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -0,0 +1,357 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.data.util.Pair; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.profile.TbTenantProfileCache; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.InternalTelemetryService; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultTbApiUsageStateService implements TbApiUsageStateService { + + public static final String HOURLY = "Hourly"; + public static final FutureCallback VOID_CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Integer result) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + private final TbClusterService clusterService; + private final PartitionService partitionService; + private final TenantService tenantService; + private final TimeseriesService tsService; + private final ApiUsageStateService apiUsageStateService; + private final SchedulerComponent scheduler; + private final TbTenantProfileCache tenantProfileCache; + + @Lazy + @Autowired + private InternalTelemetryService tsWsService; + + // Tenants that should be processed on this server + private final Map myTenantStates = new ConcurrentHashMap<>(); + // Tenants that should be processed on other servers + private final Map otherTenantStates = new ConcurrentHashMap<>(); + + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + + @Value("${usage.stats.check.cycle:60000}") + private long nextCycleCheckInterval; + + private final Lock updateLock = new ReentrantLock(); + + public DefaultTbApiUsageStateService(TbClusterService clusterService, + PartitionService partitionService, + TenantService tenantService, + TimeseriesService tsService, + ApiUsageStateService apiUsageStateService, + SchedulerComponent scheduler, + TbTenantProfileCache tenantProfileCache) { + this.clusterService = clusterService; + this.partitionService = partitionService; + this.tenantService = tenantService; + this.tsService = tsService; + this.apiUsageStateService = apiUsageStateService; + this.scheduler = scheduler; + this.tenantProfileCache = tenantProfileCache; + } + + @PostConstruct + public void init() { + if (enabled) { + log.info("Starting api usage service."); + initStatesFromDataBase(); + scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS); + } + } + + @Override + public void process(TbProtoQueueMsg msg, TbCallback callback) { + ToUsageStatsServiceMsg statsMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); + TenantApiUsageState tenantState; + List updatedEntries; + Map result = new HashMap<>(); + updateLock.lock(); + try { + tenantState = getOrFetchState(tenantId); + long ts = tenantState.getCurrentCycleTs(); + long hourTs = tenantState.getCurrentHourTs(); + long newHourTs = SchedulerUtils.getStartOfCurrentHour(); + if (newHourTs != hourTs) { + tenantState.setHour(newHourTs); + } + updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); + for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { + ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); + long newValue = tenantState.add(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue))); + long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue))); + Pair update = tenantState.checkStateUpdatedDueToThreshold(recordKey); + if (update != null) { + result.put(update.getFirst(), update.getSecond()); + } + } + } finally { + updateLock.unlock(); + } + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK); + if (!result.isEmpty()) { + persistAndNotify(tenantState, result); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { + myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + initStatesFromDataBase(); + } + } + + @Override + public ApiUsageState getApiUsageState(TenantId tenantId) { + TenantApiUsageState tenantState = myTenantStates.get(tenantId); + if (tenantState != null) { + return tenantState.getApiUsageState(); + } else { + ApiUsageState state = otherTenantStates.get(tenantId); + if (state != null) { + return state; + } else { + if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { + return getOrFetchState(tenantId).getApiUsageState(); + } else { + updateLock.lock(); + try { + state = otherTenantStates.get(tenantId); + if (state == null) { + state = apiUsageStateService.findTenantApiUsageState(tenantId); + otherTenantStates.put(tenantId, state); + } + } finally { + updateLock.unlock(); + } + return state; + } + } + } + } + + @Override + public void onApiUsageStateUpdate(TenantId tenantId) { + otherTenantStates.remove(tenantId); + } + + @Override + public void onTenantProfileUpdate(TenantProfileId tenantProfileId) { + TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); + updateLock.lock(); + try { + myTenantStates.values().forEach(state -> { + if (tenantProfile.getId().equals(state.getTenantProfileId())) { + updateTenantState(state, tenantProfile); + } + }); + } finally { + updateLock.unlock(); + } + } + + @Override + public void onTenantUpdate(TenantId tenantId) { + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + updateLock.lock(); + try { + TenantApiUsageState state = myTenantStates.get(tenantId); + if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) { + updateTenantState(state, tenantProfile); + } + } finally { + updateLock.unlock(); + } + } + + private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { + TenantProfileData oldProfileData = state.getTenantProfileData(); + state.setTenantProfileId(tenantProfile.getId()); + state.setTenantProfileData(tenantProfile.getProfileData()); + Map result = state.checkStateUpdatedDueToThresholds(); + if (!result.isEmpty()) { + persistAndNotify(state, result); + } + updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(), + oldProfileData.getConfiguration(), tenantProfile.getProfileData().getConfiguration()); + } + + private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id, + TenantProfileConfiguration oldData, TenantProfileConfiguration newData) { + long ts = System.currentTimeMillis(); + List profileThresholds = new ArrayList<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + long newProfileThreshold = newData.getProfileThreshold(key); + if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) { + log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold); + profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold))); + } + } + if (!profileThresholds.isEmpty()) { + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK); + } + } + + private void persistAndNotify(TenantApiUsageState state, Map result) { + log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result); + apiUsageStateService.update(state.getApiUsageState()); + clusterService.onApiStateChange(state.getApiUsageState(), null); + long ts = System.currentTimeMillis(); + List stateTelemetry = new ArrayList<>(); + result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState))))); + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK); + } + + private void checkStartOfNextCycle() { + updateLock.lock(); + try { + long now = System.currentTimeMillis(); + myTenantStates.values().forEach(state -> { + if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) { + state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth()); + } + }); + } finally { + updateLock.unlock(); + } + } + + private TenantApiUsageState getOrFetchState(TenantId tenantId) { + TenantApiUsageState tenantState = myTenantStates.get(tenantId); + if (tenantState == null) { + ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + if (dbStateEntity == null) { + try { + dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId); + } catch (Exception e) { + dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + } + } + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity); + try { + List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + boolean cycleEntryFound = false; + boolean hourlyEntryFound = false; + for (TsKvEntry tsKvEntry : dbValues) { + if (tsKvEntry.getKey().equals(key.getApiCountKey())) { + cycleEntryFound = true; + tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L); + } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { + hourlyEntryFound = true; + tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L); + } + if (cycleEntryFound && hourlyEntryFound) { + break; + } + } + } + log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity); + myTenantStates.put(tenantId, tenantState); + } catch (InterruptedException | ExecutionException e) { + log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); + } + } + return tenantState; + } + + private void initStatesFromDataBase() { + try { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); + for (Tenant tenant : tenantIterator) { + if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { + updateLock.lock(); + try { + updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); + } catch (Exception e) { + log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); + } finally { + updateLock.unlock(); + } + } + } + log.info("Api usage service started."); + } catch (Exception e) { + log.warn("Unknown failure", e); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java new file mode 100644 index 0000000000..e19c0a4154 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; + +public interface TbApiUsageStateService extends ApplicationListener { + + void process(TbProtoQueueMsg msg, TbCallback callback); + + ApiUsageState getApiUsageState(TenantId tenantId); + + void onTenantProfileUpdate(TenantProfileId tenantProfileId); + + void onTenantUpdate(TenantId tenantId); + + void onApiUsageStateUpdate(TenantId tenantId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java new file mode 100644 index 0000000000..8fcc1d46cb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -0,0 +1,201 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class TenantApiUsageState { + + private final Map currentCycleValues = new ConcurrentHashMap<>(); + private final Map currentHourValues = new ConcurrentHashMap<>(); + + @Getter + @Setter + private TenantProfileId tenantProfileId; + @Getter + @Setter + private TenantProfileData tenantProfileData; + @Getter + private final ApiUsageState apiUsageState; + @Getter + private volatile long currentCycleTs; + @Getter + private volatile long nextCycleTs; + @Getter + private volatile long currentHourTs; + + public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) { + this.tenantProfileId = tenantProfile.getId(); + this.tenantProfileData = tenantProfile.getProfileData(); + this.apiUsageState = apiUsageState; + this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); + this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); + this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); + } + + public void put(ApiUsageRecordKey key, Long value) { + currentCycleValues.put(key, value); + } + + public void putHourly(ApiUsageRecordKey key, Long value) { + currentHourValues.put(key, value); + } + + public long add(ApiUsageRecordKey key, long value) { + long result = currentCycleValues.getOrDefault(key, 0L) + value; + currentCycleValues.put(key, result); + return result; + } + + public long get(ApiUsageRecordKey key) { + return currentCycleValues.getOrDefault(key, 0L); + } + + public long addToHourly(ApiUsageRecordKey key, long value) { + long result = currentHourValues.getOrDefault(key, 0L) + value; + currentHourValues.put(key, result); + return result; + } + + public void setHour(long currentHourTs) { + this.currentHourTs = currentHourTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentHourValues.put(key, 0L); + } + } + + public void setCycles(long currentCycleTs, long nextCycleTs) { + this.currentCycleTs = currentCycleTs; + this.nextCycleTs = nextCycleTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentCycleValues.put(key, 0L); + } + } + + public long getProfileThreshold(ApiUsageRecordKey key) { + return tenantProfileData.getConfiguration().getProfileThreshold(key); + } + + public TenantId getTenantId() { + return apiUsageState.getTenantId(); + } + + public boolean isTransportEnabled() { + return apiUsageState.isTransportEnabled(); + } + + public boolean isDbStorageEnabled() { + return apiUsageState.isDbStorageEnabled(); + } + + public boolean isRuleEngineEnabled() { + return apiUsageState.isReExecEnabled(); + } + + public boolean isJsExecEnabled() { + return apiUsageState.isJsExecEnabled(); + } + + public void setTransportEnabled(boolean transportEnabled) { + apiUsageState.setTransportEnabled(transportEnabled); + } + + public void setDbStorageEnabled(boolean dbStorageEnabled) { + apiUsageState.setDbStorageEnabled(dbStorageEnabled); + } + + public void setRuleEngineEnabled(boolean ruleEngineEnabled) { + apiUsageState.setReExecEnabled(ruleEngineEnabled); + } + + public void setJsExecEnabled(boolean jsExecEnabled) { + apiUsageState.setJsExecEnabled(jsExecEnabled); + } + + public boolean isFeatureEnabled(ApiUsageRecordKey recordKey) { + switch (recordKey) { + case TRANSPORT_MSG_COUNT: + case TRANSPORT_DP_COUNT: + return isTransportEnabled(); + case RE_EXEC_COUNT: + return isRuleEngineEnabled(); + case STORAGE_DP_COUNT: + return isDbStorageEnabled(); + case JS_EXEC_COUNT: + return isJsExecEnabled(); + default: + return true; + } + } + + public ApiFeature setFeatureValue(ApiUsageRecordKey recordKey, boolean value) { + ApiFeature feature = null; + boolean currentValue = isFeatureEnabled(recordKey); + switch (recordKey) { + case TRANSPORT_MSG_COUNT: + case TRANSPORT_DP_COUNT: + feature = ApiFeature.TRANSPORT; + setTransportEnabled(value); + break; + case RE_EXEC_COUNT: + feature = ApiFeature.RE; + setRuleEngineEnabled(value); + break; + case STORAGE_DP_COUNT: + feature = ApiFeature.DB; + setDbStorageEnabled(value); + break; + case JS_EXEC_COUNT: + feature = ApiFeature.JS; + setJsExecEnabled(value); + break; + } + return currentValue == value ? null : feature; + } + + public Map checkStateUpdatedDueToThresholds() { + Map result = new HashMap<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + Pair featureUpdate = checkStateUpdatedDueToThreshold(key); + if (featureUpdate != null) { + result.put(featureUpdate.getFirst(), featureUpdate.getSecond()); + } + } + return result; + } + + public Pair checkStateUpdatedDueToThreshold(ApiUsageRecordKey recordKey) { + long value = get(recordKey); + long threshold = getProfileThreshold(recordKey); + boolean featureValue = threshold == 0 || value < threshold; + ApiFeature feature = setFeatureValue(recordKey, featureValue); + return feature != null ? Pair.of(feature, featureValue) : null; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index aa2175d260..fd757ed6dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; @@ -127,6 +128,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { public void createDefaultTenantProfiles() throws Exception { tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + TenantProfile isolatedTbCoreProfile = new TenantProfile(); isolatedTbCoreProfile.setDefault(false); isolatedTbCoreProfile.setName("Isolated TB Core"); @@ -134,6 +138,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); isolatedTbCoreProfile.setIsolatedTbCore(true); isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); + isolatedTbCoreProfile.setProfileData(tenantProfileData); try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); } catch (DataValidationException e) { @@ -147,6 +152,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); isolatedTbRuleEngineProfile.setIsolatedTbCore(false); isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); } catch (DataValidationException e) { @@ -160,6 +167,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); } catch (DataValidationException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java index 6784405adb..3db23c65e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java @@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; import java.util.function.Consumer; @Service @@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { private final ConcurrentMap deviceProfilesMap = new ConcurrentHashMap<>(); private final ConcurrentMap devicesMap = new ConcurrentHashMap<>(); - private final ConcurrentMap>> listeners = new ConcurrentHashMap<>(); + private final ConcurrentMap>> profileListeners = new ConcurrentHashMap<>(); + private final ConcurrentMap>> deviceProfileListeners = new ConcurrentHashMap<>(); public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { this.deviceProfileService = deviceProfileService; @@ -87,33 +88,37 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { return get(tenantId, profileId); } - @Override - public void put(DeviceProfile profile) { - if (profile.getId() != null) { - deviceProfilesMap.put(profile.getId(), profile); - log.debug("[{}] pushed device profile to cache: {}", profile.getId(), profile); - notifyListeners(profile); - } - } - @Override public void evict(TenantId tenantId, DeviceProfileId profileId) { DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); DeviceProfile newProfile = get(tenantId, profileId); if (newProfile != null) { - notifyListeners(newProfile); + notifyProfileListeners(newProfile); } } @Override - public void evict(DeviceId deviceId) { - devicesMap.remove(deviceId); + public void evict(TenantId tenantId, DeviceId deviceId) { + DeviceProfileId old = devicesMap.remove(deviceId); + if (old != null) { + DeviceProfile newProfile = get(tenantId, deviceId); + if (newProfile == null || !old.equals(newProfile.getId())) { + notifyDeviceListeners(tenantId, deviceId, newProfile); + } + } } @Override - public void addListener(TenantId tenantId, EntityId listenerId, Consumer listener) { - listeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, listener); + public void addListener(TenantId tenantId, EntityId listenerId, + Consumer profileListener, + BiConsumer deviceListener) { + if (profileListener != null) { + profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener); + } + if (deviceListener != null) { + deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener); + } } @Override @@ -128,17 +133,30 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { @Override public void removeListener(TenantId tenantId, EntityId listenerId) { - ConcurrentMap> tenantListeners = listeners.get(tenantId); + ConcurrentMap> tenantListeners = profileListeners.get(tenantId); if (tenantListeners != null) { tenantListeners.remove(listenerId); } + ConcurrentMap> deviceListeners = deviceProfileListeners.get(tenantId); + if (deviceListeners != null) { + deviceListeners.remove(listenerId); + } } - private void notifyListeners(DeviceProfile profile) { - ConcurrentMap> tenantListeners = listeners.get(profile.getTenantId()); + private void notifyProfileListeners(DeviceProfile profile) { + ConcurrentMap> tenantListeners = profileListeners.get(profile.getTenantId()); if (tenantListeners != null) { tenantListeners.forEach((id, listener) -> listener.accept(profile)); } } + private void notifyDeviceListeners(TenantId tenantId, DeviceId deviceId, DeviceProfile profile) { + if (profile != null) { + ConcurrentMap> tenantListeners = deviceProfileListeners.get(tenantId); + if (tenantListeners != null) { + tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile)); + } + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java index e65f297d53..10e28bd299 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java @@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { - void put(DeviceProfile profile); - void evict(TenantId tenantId, DeviceProfileId id); - void evict(DeviceId id); + void evict(TenantId tenantId, DeviceId id); DeviceProfile find(DeviceProfileId deviceProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 9a331bb208..18f064b745 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; @@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -214,6 +216,12 @@ public class DefaultTbClusterService implements TbClusterService { onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback); } + @Override + public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) { + onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback); + broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED)); + } + @Override public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); @@ -229,13 +237,14 @@ public class DefaultTbClusterService implements TbClusterService { onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); } - public void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { - log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entity.getName()); + public void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { + String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName(); + log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName); TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() .setEntityType(entityid.getEntityType().name()) .setData(ByteString.copyFrom(encodingService.encode(entity))).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); - broadcast(transportMsg); + broadcast(transportMsg, callback); } private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { @@ -246,15 +255,16 @@ public class DefaultTbClusterService implements TbClusterService { .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); - broadcast(transportMsg); + broadcast(transportMsg, callback); } - private void broadcast(ToTransportMsg transportMsg) { + private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) { TbQueueProducer> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); Set tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); + TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null; for (String transportServiceId : tbTransportServices) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); - toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); + toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback); toTransportNfs.incrementAndGet(); } } @@ -264,7 +274,8 @@ public class DefaultTbClusterService implements TbClusterService { TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) - || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)) { + || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE) + || msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 5d240790f2..aaafa52c35 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -17,41 +17,44 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; @@ -70,6 +73,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -88,32 +93,56 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> mainConsumer; private final DeviceStateService stateService; + private final TbApiUsageStateService statsService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats; + protected final TbQueueConsumer> usageStatsConsumer; + + protected volatile ExecutorService usageStatsExecutor; - public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, - DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, - SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, - TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { - super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, + ActorSystemContext actorContext, + DeviceStateService stateService, + TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, + DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService, + StatsFactory statsFactory, + TbDeviceProfileCache deviceProfileCache, + TbApiUsageStateService statsService, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer(); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; this.stats = new TbCoreConsumerStats(statsFactory); + this.statsService = statsService; } @PostConstruct public void init() { super.init("tb-core-consumer", "tb-core-notifications-consumer"); + this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer")); } @PreDestroy public void destroy() { super.destroy(); + if (usageStatsExecutor != null) { + usageStatsExecutor.shutdownNow(); + } + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + launchUsageStatsConsumer(); } @Override @@ -121,6 +150,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService tpi.newByTopic(usageStatsConsumer.getTopic())) + .collect(Collectors.toSet())); } } @@ -223,6 +258,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + while (!stopped) { + try { + List> msgs = usageStatsConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + handleUsageStats(msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process usge stats: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue())); + } + usageStatsConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain usage stats from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new usage stats", e2); + } + } + } + } + log.info("TB Usage Stats Consumer stopped."); + }); + } + + private void handleUsageStats(TbProtoQueueMsg msg, TbCallback callback) { + statsService.process(msg, callback); + } + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) @@ -321,6 +403,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> nfConsumer; public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, - TbDeviceProfileCache deviceProfileCache, TbQueueConsumer> nfConsumer) { + TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer> nfConsumer) { this.actorContext = actorContext; this.encodingService = encodingService; + this.tenantProfileCache = tenantProfileCache; this.deviceProfileCache = deviceProfileCache; + this.apiUsageStateService = apiUsageStateService; this.nfConsumer = nfConsumer; } @@ -143,10 +151,23 @@ public abstract class AbstractConsumerService scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map disabledFunctions = new ConcurrentHashMap<>(); + + protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + this.apiUsageStateService = apiUsageStateService; + this.apiUsageClient = apiUsageClient; + } public void init(long maxRequestsTimeout) { if (maxRequestsTimeout > 0) { @@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } @Override - public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { - UUID scriptId = UUID.randomUUID(); - String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); - String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); - return doEval(scriptId, functionName, jsScript); + public ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) { + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + UUID scriptId = UUID.randomUUID(); + String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); + String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); + return doEval(scriptId, functionName, jsScript); + } else { + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); + } } @Override - public ListenableFuture invokeFunction(UUID scriptId, Object... args) { - String functionName = scriptIdToNameMap.get(scriptId); - if (functionName == null) { - return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); - } - if (!isBlackListed(scriptId)) { - return doInvokeFunction(scriptId, functionName, args); + public ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args) { + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + String functionName = scriptIdToNameMap.get(scriptId); + if (functionName == null) { + return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); + } + if (!isDisabled(scriptId)) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); + return doInvokeFunction(scriptId, functionName, args); + } else { + return Futures.immediateFailedFuture( + new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!")); + } } else { - return Futures.immediateFailedFuture( - new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!")); + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); } } @@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { if (functionName != null) { try { scriptIdToNameMap.remove(scriptId); - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); doRelease(scriptId, functionName); } catch (Exception e) { return Futures.immediateFailedFuture(e); @@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { @@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } - private boolean isBlackListed(UUID scriptId) { - BlackListInfo errorCount = blackListedFunctions.get(scriptId); + private boolean isDisabled(UUID scriptId) { + DisableListInfo errorCount = disabledFunctions.get(scriptId); if (errorCount != null) { if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); return false; } else { return errorCount.get() >= getMaxErrors(); @@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } } - private class BlackListInfo { + private class DisableListInfo { private final AtomicInteger counter; private long expirationTime; - private BlackListInfo() { + private DisableListInfo() { this.counter = new AtomicInteger(0); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 8e6666bbba..db8bb68b4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes; import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,7 +38,6 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); - @Autowired @Getter - private JsExecutorService jsExecutor; + private final JsExecutorService jsExecutor; @Value("${js.local.max_requests_timeout:0}") private long maxRequestsTimeout; @@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @Value("${js.local.stats.enabled:false}") private boolean statsEnabled; + public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient); + this.jsExecutor = jsExecutor; + } + @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java index 97e40e6f62..15fd8c7534 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java @@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor { @Override protected int getThreadPollSize() { - return jsExecutorThreadPoolSize; + return Math.max(jsExecutorThreadPoolSize, 1); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java index a9f727e4a7..ae990af66c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java @@ -17,14 +17,15 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; public interface JsInvokeService { - ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames); + ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames); - ListenableFuture invokeFunction(UUID scriptId, Object... args); + ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args); ListenableFuture release(UUID scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 66a14cc827..0dd45d340c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import java.util.concurrent.TimeUnit; @@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; + public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient, jsExecutor); + } + @Override protected boolean useJsSandbox() { return useJsSandbox; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index cfb0c8a8ee..7b4114dce6 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); + public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + super(apiUsageStateService, apiUsageClient); + } + @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 110f410fad..275e645ce2 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private final JsInvokeService sandboxService; private final UUID scriptId; + private final TenantId tenantId; private final EntityId entityId; - public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + this.tenantId = tenantId; this.sandboxService = sandboxService; this.entityId = entityId; try { - this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); + this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); } catch (Exception e) { Throwable t = e; if (e instanceof ExecutionException) { @@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private JsonNode executeScript(TbMsg msg) throws ScriptException { try { String[] inArgs = prepareArgs(msg); - String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); + String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); return mapper.readTree(eval); } catch (ExecutionException e) { if (e.getCause() instanceof ScriptException) { @@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private ListenableFuture executeScriptAsync(TbMsg msg) { String[] inArgs = prepareArgs(msg); - return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]), o -> { try { return Futures.immediateFuture(mapper.readTree(o.toString())); diff --git a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java index b4bc79e06e..57c192235e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java +++ b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java @@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -55,6 +57,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -111,6 +114,9 @@ public class AccessValidator { @Autowired protected AccessControlService accessControlService; + @Autowired + protected ApiUsageStateService apiUsageStateService; + private ExecutorService executor; @PostConstruct @@ -193,6 +199,9 @@ public class AccessValidator { case ENTITY_VIEW: validateEntityView(currentUser, operation, entityId, callback); return; + case API_USAGE_STATE: + validateApiUsageState(currentUser, operation, entityId, callback); + return; default: //TODO: add support of other entities throw new IllegalStateException("Not Implemented!"); @@ -237,6 +246,27 @@ public class AccessValidator { } } + private void validateApiUsageState(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback callback) { + if (currentUser.isSystemAdmin()) { + callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); + } else { + if (!operation.equals(Operation.READ_TELEMETRY)) { + callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!")); + } + ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId())); + if (apiUsageState == null) { + callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!")); + } else { + try { + accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState); + } catch (ThingsboardException e) { + callback.onSuccess(ValidationResult.accessDenied(e.getMessage())); + } + callback.onSuccess(ValidationResult.ok(apiUsageState)); + } + } + } + private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback callback) { if (currentUser.isSystemAdmin()) { callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 08534dc2d4..9d16760464 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -35,7 +35,8 @@ public enum Resource { OAUTH2_CONFIGURATION_INFO(), OAUTH2_CONFIGURATION_TEMPLATE(), TENANT_PROFILE(EntityType.TENANT_PROFILE), - DEVICE_PROFILE(EntityType.DEVICE_PROFILE); + DEVICE_PROFILE(EntityType.DEVICE_PROFILE), + API_USAGE_STATE(EntityType.API_USAGE_STATE); private final EntityType entityType; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 4124292c95..e0c34200f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index b46033389b..a832eb0870 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -48,9 +48,9 @@ import java.util.stream.Collectors; public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; - public static final FutureCallback CALLBACK = new FutureCallback() { + public static final FutureCallback CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable Integer result) { } @@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .collect(Collectors.toList()); if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK); } } } catch (DataValidationException e) { @@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); } catch (DataValidationException e2) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { throw e2; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index bc4099e81b..468c3c26b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } } - protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback>() { + protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { @Override - public void onSuccess(@Nullable List result) { + public void onSuccess(@Nullable T result) { callback.accept(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index aecd888dce..7a35e6b317 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.EntityId; @@ -41,11 +40,12 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; @@ -59,12 +59,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -76,6 +72,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final AttributesService attrService; private final TimeseriesService tsService; private final EntityViewService entityViewService; + private final TbApiUsageClient apiUsageClient; + private final TbApiUsageStateService apiUsageStateService; private ExecutorService tsCallBackExecutor; @@ -83,11 +81,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, EntityViewService entityViewService, TbClusterService clusterService, - PartitionService partitionService) { + PartitionService partitionService, + TbApiUsageClient apiUsageClient, + TbApiUsageStateService apiUsageStateService) { super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; this.entityViewService = entityViewService; + this.apiUsageClient = apiUsageClient; + this.apiUsageStateService = apiUsageStateService; } @PostConstruct @@ -116,7 +118,35 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); + checkInternalEntity(entityId); + if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { + @Override + public void onSuccess(Integer result) { + if (result != null && result > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + } + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } else{ + callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); + } + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { + ListenableFuture saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { @@ -141,9 +171,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Optional tsKvEntry = entries.stream() .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) .max(Comparator.comparingLong(TsKvEntry::getTs)); - if (tsKvEntry.isPresent()) { - entityViewLatest.add(tsKvEntry.get()); - } + tsKvEntry.ifPresent(entityViewLatest::add); } } if (!entityViewLatest.isEmpty()) { @@ -176,29 +204,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { + checkInternalEntity(entityId); + saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @Override public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + checkInternalEntity(entityId); + saveLatestAndNotifyInternal(tenantId, entityId, ts, callback); + } + + @Override + public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback); + } + + @Override + public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); } @Override public void deleteLatest(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteLatestInternal(tenantId, entityId, keys, callback); + } + + @Override + public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); } @Override @@ -283,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + private void addVoidCallback(ListenableFuture saveFuture, final FutureCallback callback) { Futures.addCallback(saveFuture, new FutureCallback() { @Override public void onSuccess(@Nullable S result) { @@ -296,4 +348,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } }, tsCallBackExecutor); } + + private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable S result) { + callback.onSuccess(result); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }, tsCallBackExecutor); + } + + private void checkInternalEntity(EntityId entityId) { + if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) { + throw new RuntimeException("Can't update API Usage State!"); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java new file mode 100644 index 0000000000..1ca2cdeacb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +/** + * Created by ashvayka on 27.03.18. + */ +public interface InternalTelemetryService extends RuleEngineTelemetryService { + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); + + void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + + void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback); + + void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback); + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d652a3f1c5..a88a378cd7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { +public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 10360769a7..ab566fb147 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -25,6 +25,7 @@ import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -51,7 +52,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; @@ -68,6 +68,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.dao.device.provision.ProvisionFailedException; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbTenantProfileCache; @@ -92,6 +93,7 @@ public class DefaultTransportApiService implements TransportApiService { private final TbDeviceProfileCache deviceProfileCache; private final TbTenantProfileCache tenantProfileCache; + private final TbApiUsageStateService apiUsageStateService; private final DeviceService deviceService; private final RelationService relationService; private final DeviceCredentialsService deviceCredentialsService; @@ -104,13 +106,14 @@ public class DefaultTransportApiService implements TransportApiService { private final ConcurrentMap deviceCreationLocks = new ConcurrentHashMap<>(); public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, - TbTenantProfileCache tenantProfileCache, DeviceService deviceService, + TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService, RelationService relationService, DeviceCredentialsService deviceCredentialsService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, DeviceProvisionService deviceProvisionService) { this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; + this.apiUsageStateService = apiUsageStateService; this.deviceService = deviceService; this.relationService = relationService; this.deviceCredentialsService = deviceCredentialsService; @@ -316,18 +319,21 @@ public class DefaultTransportApiService implements TransportApiService { private ListenableFuture handle(GetEntityProfileRequestMsg requestMsg) { EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); - ByteString data; + GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder(); if (entityType.equals(EntityType.DEVICE_PROFILE)) { DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); - data = ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))); } else if (entityType.equals(EntityType.TENANT)) { - TenantProfile tenantProfile = tenantProfileCache.get(new TenantId(entityUuid)); - data = ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)); + TenantId tenantId = new TenantId(entityUuid); + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile))); + builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state))); } else { throw new RuntimeException("Invalid entity profile request: " + entityType); } - return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(GetEntityProfileResponseMsg.newBuilder().setData(data).build()).build()); + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build()); } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index deb28ffd99..f0f4b9dd54 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -673,6 +673,7 @@ queue: poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index a4d615ae2b..4f0d1e201b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName(name); tenantProfile.setDescription(name + " Test"); - tenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); tenantProfile.setDefault(false); tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbRuleEngine(false); diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 64f9b8fddf..bc2c36b501 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -27,7 +27,7 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", -// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", +// "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest", "org.thingsboard.server.controller.sql.*Test", }) public class ControllerSqlTestSuite { diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java deleted file mode 100644 index 60495b119d..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; - -import javax.script.ScriptException; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.*; - -public class RuleNodeJsScriptEngineTest { - - private ScriptEngine scriptEngine; - private TestNashornJsInvokeService jsSandboxService; - - private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased()); - - @Before - public void beforeTest() throws Exception { - jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3); - } - - @After - public void afterTest() throws Exception { - jsSandboxService.stop(); - } - - @Test - public void msgCanBeUpdated() throws ScriptException { - String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("70", actual.getMetaData().getValue("temp")); - scriptEngine.destroy(); - } - - @Test - public void newAttributesCanBeAddedInMsg() throws ScriptException { - String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("94", actual.getMetaData().getValue("newAttr")); - scriptEngine.destroy(); - } - - @Test - public void payloadCanBeUpdated() throws ScriptException { - String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - - String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}"; - assertEquals(expectedJson, actual.getData()); - scriptEngine.destroy(); - } - - @Test - public void metadataAccessibleForFilter() throws ScriptException { - String function = "return metadata.humidity < 15;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - assertFalse(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForFilter() throws ScriptException { - String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); - assertTrue(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return 'one'\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one"), actual); - scriptEngine.destroy(); - } - - @Test - public void multipleRelationsReturnedFromSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return ['three', 'one']\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one", "three"), actual); - scriptEngine.destroy(); - } - - @Test - public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException { - String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - - int repeat = 1000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - } - - finishLatch.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - - CountDownLatch nextStart = new CountDownLatch(repeat); - CountDownLatch nextFinish = new CountDownLatch(repeat); - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code)); - } - - nextFinish.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - service.shutdownNow(); - } - - @Test - public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException { - String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};"; - - int repeat = 10000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - for (int i = 0; i < repeat; i++) { - service.submit(() -> { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - }); - } - - finishLatch.await(); - assertTrue(scriptIds.isEmpty()); - assertEquals(repeat, failedCount.get()); - service.shutdownNow(); - } - - private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount, - Map scriptIds, String code) { - try { - for (int k = 0; k < 10; k++) { - startLatch.countDown(); - startLatch.await(); - UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get(); - scriptIds.put(scriptId, new Object()); - jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get(); - jsSandboxService.release(scriptId).get(); - } - } catch (Throwable th) { - failedCount.incrementAndGet(); - } finally { - finishLatch.countDown(); - } - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java deleted file mode 100644 index 8a89de5dff..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { - - private boolean useJsSandbox; - private final int monitorThreadPoolSize; - private final long maxCpuTime; - private final int maxErrors; - - public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) { - this.useJsSandbox = useJsSandbox; - this.monitorThreadPoolSize = monitorThreadPoolSize; - this.maxCpuTime = maxCpuTime; - this.maxErrors = maxErrors; - init(); - } - - @Override - protected boolean useJsSandbox() { - return useJsSandbox; - } - - @Override - protected int getMonitorThreadPoolSize() { - return monitorThreadPoolSize; - } - - @Override - protected long getMaxCpuTime() { - return maxCpuTime; - } - - @Override - protected int getMaxErrors() { - return maxErrors; - } - - @Override - protected long getMaxBlacklistDuration() { - return 100000; - } -} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java index d5ad9bbbb6..924cad0884 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java @@ -31,4 +31,6 @@ public interface RuleNodeStateService { RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); + + void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 2a5a701f9e..0e11aedaf5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -36,9 +36,9 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java new file mode 100644 index 0000000000..5388cab283 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ApiUsageStateService { + + ApiUsageState createDefaultApiUsageState(TenantId id); + + ApiUsageState update(ApiUsageState apiUsageState); + + ApiUsageState findTenantApiUsageState(TenantId tenantId); + + void deleteApiUsageStateByTenantId(TenantId tenantId); + + ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java new file mode 100644 index 0000000000..b1ff230e71 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Getter; + +public enum ApiUsageRecordKey { + + TRANSPORT_MSG_COUNT("transportMsgCount", "transportMsgLimit"), + TRANSPORT_DP_COUNT("transportDataPointsCount", "transportDataPointsLimit"), + STORAGE_DP_COUNT("storageDataPointsCount", "storageDataPointsLimit"), + RE_EXEC_COUNT("ruleEngineExecutionCount", "ruleEngineExecutionLimit"), + JS_EXEC_COUNT("jsExecutionCount", "jsExecutionLimit"); + + @Getter + private final String apiCountKey; + @Getter + private final String apiLimitKey; + + ApiUsageRecordKey(String apiCountKey, String apiLimitKey) { + this.apiCountKey = apiCountKey; + this.apiLimitKey = apiLimitKey; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java new file mode 100644 index 0000000000..5144861790 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class ApiUsageState extends BaseData implements HasTenantId { + + private static final long serialVersionUID = 8250339805336035966L; + + @Getter + @Setter + private TenantId tenantId; + @Getter + @Setter + private EntityId entityId; + @Getter + @Setter + private boolean transportEnabled = true; + @Getter + @Setter + private boolean dbStorageEnabled = true; + @Getter + @Setter + private boolean reExecEnabled = true; + @Getter + @Setter + private boolean jsExecEnabled = true; + + public ApiUsageState() { + super(); + } + + public ApiUsageState(ApiUsageStateId id) { + super(id); + } + + public ApiUsageState(ApiUsageState ur) { + super(ur); + this.tenantId = ur.getTenantId(); + this.entityId = ur.getEntityId(); + this.transportEnabled = ur.isTransportEnabled(); + this.dbStorageEnabled = ur.isDbStorageEnabled(); + this.reExecEnabled = ur.isReExecEnabled(); + this.jsExecEnabled = ur.isJsExecEnabled(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index cfe12a14cb..9f7fb711e8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data; * @author Andrew Shvayka */ public enum EntityType { - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 32f620f869..acb9c239b8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -21,6 +21,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased implements H profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); } catch (IOException e) { log.warn("Can't deserialize tenant profile data: ", e); - return null; + return createDefaultTenantProfileData(); } return profileData; } else { - return null; + return createDefaultTenantProfileData(); } } } + public TenantProfileData createDefaultTenantProfileData() { + TenantProfileData tpd = new TenantProfileData(); + tpd.setConfiguration(new DefaultTenantProfileConfiguration()); + return tpd; + } + public void setProfileData(TenantProfileData data) { this.profileData = data; try { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java similarity index 53% rename from common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java index 54319276f1..2b0ed502ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java @@ -15,28 +15,6 @@ */ package org.thingsboard.server.common.data; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Data; - -import java.util.HashMap; -import java.util.Map; - -@Data -public class TenantProfileData { - - @JsonIgnore - private Map properties = new HashMap<>(); - - @JsonAnyGetter - public Map properties() { - return this.properties; - } - - @JsonAnySetter - public void put(String name, Object value) { - this.properties.put(name, value); - } - +public enum TenantProfileType { + DEFAULT } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java index b6437ae7bb..17035f337b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +28,7 @@ public class DeviceProfileAlarm { private String id; private String alarmType; - private Map createRules; + private LinkedHashMap createRules; private AlarmRule clearRule; // Hidden in advanced settings diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java new file mode 100644 index 0000000000..87983f519a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public class ApiUsageStateId extends UUIDBased implements EntityId { + + @JsonCreator + public ApiUsageStateId(@JsonProperty("id") UUID id) { + super(id); + } + + public static ApiUsageStateId fromString(String userId) { + return new ApiUsageStateId(UUID.fromString(userId)); + } + + @JsonIgnore + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index dc1589814f..8a1a769c5e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -66,6 +66,8 @@ public class EntityIdFactory { return new DeviceProfileId(uuid); case TENANT_PROFILE: return new TenantProfileId(uuid); + case API_USAGE_STATE: + return new ApiUsageStateId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index c2d6688004..bcf49899dc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -19,7 +19,7 @@ import java.util.Objects; import java.util.Optional; public class BasicTsKvEntry implements TsKvEntry { - + private static final int MAX_CHARS_PER_DATA_POINT = 512; private final long ts; private final KvEntry kv; @@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry { public String getValueAsString() { return kv.getValueAsString(); } + + @Override + public int getDataPoints() { + int length; + switch (getDataType()) { + case STRING: + length = getStrValue().get().length(); + break; + case JSON: + length = getJsonValue().get().length(); + break; + default: + return 1; + } + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index 315d3940df..e2f8eaf1e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * Represents time series KV data entry * @@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry { long getTs(); + @JsonIgnore + int getDataPoints(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index 87adc31713..1f242a0f88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -21,5 +21,5 @@ import java.io.Serializable; * @author Andrew Shvayka */ public enum ComponentLifecycleEvent implements Serializable { - CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED + CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, ADDED_TO_ALLOW_LIST, ADDED_TO_DENY_LIST } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java new file mode 100644 index 0000000000..436aa44230 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class ApiUsageStateFilter implements EntityFilter { + @Override + public EntityFilterType getType() { + return EntityFilterType.API_USAGE_STATE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index d3b9c39530..f2a6dde626 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), + @JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"), @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java index 0c05fdc8e4..12860b8453 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java @@ -25,7 +25,8 @@ public enum EntityFilterType { RELATIONS_QUERY("relationsQuery"), ASSET_SEARCH_QUERY("assetSearchQuery"), DEVICE_SEARCH_QUERY("deviceSearchQuery"), - ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); + ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"), + API_USAGE_STATE("apiUsageState"); private final String label; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java new file mode 100644 index 0000000000..1804731c04 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfileType; + +@Data +public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration { + + private long maxDevices; + private long maxAssets; + + private String transportTenantMsgRateLimit; + private String transportTenantTelemetryMsgRateLimit; + private String transportTenantTelemetryDataPointsRateLimit; + private String transportDeviceMsgRateLimit; + private String transportDeviceTelemetryMsgRateLimit; + private String transportDeviceTelemetryDataPointsRateLimit; + + private long maxTransportMessages; + private long maxTransportDataPoints; + private long maxREExecutions; + private long maxJSExecutions; + private long maxDPStorageDays; + private int maxRuleNodeExecutionsPerMessage; + + @Override + public long getProfileThreshold(ApiUsageRecordKey key) { + switch (key) { + case TRANSPORT_MSG_COUNT: + return maxTransportMessages; + case TRANSPORT_DP_COUNT: + return maxTransportDataPoints; + case JS_EXEC_COUNT: + return maxJSExecutions; + case RE_EXEC_COUNT: + return maxREExecutions; + case STORAGE_DP_COUNT: + return maxDPStorageDays; + } + return 0L; + } + + + @Override + public TenantProfileType getType() { + return TenantProfileType.DEFAULT; + } + + @Override + public int getMaxRuleNodeExecsPerMessage() { + return maxRuleNodeExecutionsPerMessage; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java new file mode 100644 index 0000000000..c4c6d7563a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.TenantProfileType; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")}) +public interface TenantProfileConfiguration { + + @JsonIgnore + TenantProfileType getType(); + + @JsonIgnore + long getProfileThreshold(ApiUsageRecordKey key); + + @JsonIgnore + int getMaxRuleNodeExecsPerMessage(); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java similarity index 71% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java rename to common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index 54d83610c2..297e562d26 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.limits; +package org.thingsboard.server.common.data.tenant.profile; -public interface TransportRateLimitFactory { +import lombok.Data; - TransportRateLimit create(TransportRateLimitType type, Object config); +@Data +public class TenantProfileData { - TransportRateLimit createDefault(TransportRateLimitType type); + private TenantProfileConfiguration configuration; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 844a4d0216..0b257d9d6e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import lombok.AccessLevel; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import java.io.IOException; import java.io.Serializable; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by ashvayka on 13.01.18. */ @Data -@Builder @Slf4j public final class TbMsg implements Serializable { @@ -52,61 +52,73 @@ public final class TbMsg implements Serializable { private final String data; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; + @Getter(value = AccessLevel.NONE) + private final AtomicInteger ruleNodeExecCounter; + + public int getAndIncrementRuleNodeCounter() { + return ruleNodeExecCounter.getAndIncrement(); + } + //This field is not serialized because we use queues and there is no need to do it @JsonIgnore transient private final TbMsgCallback callback; public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, + metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } + // REALLY NEW MSG + public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY); } + // For Tests only + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback); } - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(), + data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) { - return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, - origMsg.data, ruleChainId, null, origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) { + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, String queueName) { - return new TbMsg(queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, - origMsg.data, origMsg.getRuleChainId(), null, origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, String queueName) { + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId, String queueName) { - return new TbMsg(queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, - origMsg.data, ruleChainId, null, origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) { + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY); } private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) { this.id = id; this.queueName = queueName; if (ts > 0) { @@ -121,6 +133,7 @@ public final class TbMsg implements Serializable { this.data = data; this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; + this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter); if (callback != null) { this.callback = callback; } else { @@ -157,6 +170,7 @@ public final class TbMsg implements Serializable { builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); + builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get()); return builder.build().toByteArray(); } @@ -174,18 +188,18 @@ public final class TbMsg implements Serializable { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback); } public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback); } public TbMsgCallback getCallback() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index bf0fffe404..c2dec02612 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -50,6 +50,10 @@ public class TopicPartitionInfo { this.fullTopicName = tmp; } + public TopicPartitionInfo newByTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + } + public String getTopic() { return topic; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java index fea6fcb337..421066b30b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java @@ -15,16 +15,64 @@ */ package org.thingsboard.server.common.msg.tools; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjuster; +import java.time.temporal.TemporalAdjusters; +import java.time.temporal.TemporalUnit; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoUnit.MONTHS; + public class SchedulerUtils { + private final static ZoneId UTC = ZoneId.of("UTC"); private static final ConcurrentMap tzMap = new ConcurrentHashMap<>(); public static ZoneId getZoneId(String tz) { return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); } + public static long getStartOfCurrentHour() { + return getStartOfCurrentHour(UTC); + } + + public static long getStartOfCurrentHour(ZoneId zoneId) { + return LocalDateTime.now(ZoneOffset.UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli(); + } + + public static long getStartOfCurrentMonth() { + return getStartOfCurrentMonth(UTC); + } + + public static long getStartOfCurrentMonth(ZoneId zoneId) { + return LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextMonth() { + return getStartOfNextMonth(UTC); + } + + public static long getStartOfNextMonth(ZoneId zoneId) { + return LocalDate.now().with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextNextMonth() { + return getStartOfNextNextMonth(UTC); + } + + public static long getStartOfNextNextMonth(ZoneId zoneId) { + return LocalDate.now().with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static TemporalAdjuster firstDayOfNextNextMonth() { + return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS); + } + } diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 25e006121f..53cb90014b 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -45,4 +45,5 @@ message TbMsgProto { string data = 14; int64 ts = 15; + int32 ruleNodeExecCounter = 16; } \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java new file mode 100644 index 0000000000..c28f3637b7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbQueueCallbackWrapper implements TbQueueCallback { + + private final AtomicInteger tbQueueCallbackCount; + private final TbQueueCallback callback; + + public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) { + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (tbQueueCallbackCount.decrementAndGet() <= 0) { + callback.onSuccess(metadata); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index c1d724c207..0299720005 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -22,7 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 4889a7182d..4625b9d4fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index cf43bc5fe4..851b8cc40e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory notificationAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 8106ee084b..103711f36b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 8ebda6e7b9..f7a47ecf3a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); } + + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 9e8e326f0d..9ddf72534c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory public TbQueueConsumer> createTransportNotificationsConsumer() { return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 31ef4d3e45..538d91aa0f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index bf2ec6751e..5e1b398e91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index dc44caf3f0..118f0f3316 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 1cf8bca4b2..7961bb97bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { return responseBuilder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 29a60af1e2..d7c7542626 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 7d85b5c3ba..71a1ceaf9b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index a6105fea0c..ad679039e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -146,6 +147,11 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 72665c5163..efde055271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -114,6 +115,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1f812ea13c..e6eff652be 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -22,7 +22,14 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +98,64 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +182,17 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 65d582c4bc..e3064dd62d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -163,6 +164,17 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 26abf78164..a1097049db 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -144,6 +145,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index 03cd6eb557..b28a2cb7e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -111,6 +112,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3db6496b7d..a205f9b061 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -180,6 +181,17 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index a3ba577f06..f00aee331b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -101,7 +103,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -111,7 +113,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -122,10 +124,10 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -162,6 +164,17 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index e4f6dbfbb4..40a183a254 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -93,7 +95,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -103,7 +105,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -114,10 +116,10 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -143,6 +145,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index 1e900f37e5..4781d46583 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -18,19 +18,24 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -69,17 +74,17 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumerTemplate = + TbQueueConsumer> consumerTemplate = new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); @@ -90,20 +95,25 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 59a8f449b0..7c19a1ba2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -33,7 +34,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -77,6 +78,13 @@ public interface TbCoreQueueFactory { */ TbQueueConsumer> createToCoreMsgConsumer(); + /** + * Used to consume messages about usage statistics by TB Core Service + * + * @return + */ + TbQueueConsumer> createToUsageStatsServiceMsgConsumer(); + /** * Used to consume high priority messages by TB Core Service * diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9fe5de0bcc..e620259960 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,11 +33,12 @@ import javax.annotation.PostConstruct; public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private final TbCoreQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -45,30 +51,36 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 34c6ee577f..8c3a0b6565 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -63,4 +64,10 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbUsageStatsMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 69de7ca3e2..309def1269 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,12 +33,12 @@ import javax.annotation.PostConstruct; public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private final TbRuleEngineQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -46,30 +51,36 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 561b3e84ea..2f80bbce45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -32,7 +32,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index dc1d2c449c..bd6c6e0584 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -15,17 +15,17 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbTransportQueueFactory { +public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index bced3f83ac..dee6508ef4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,8 +33,9 @@ import javax.annotation.PostConstruct; public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toUsageStats; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -39,30 +45,36 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public void init() { this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java new file mode 100644 index 0000000000..f2ce1a9fe5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TbUsageStatsClientQueueFactory { + + TbQueueProducer> createToUsageStatsServiceMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java new file mode 100644 index 0000000000..02ce60496e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +@Component +public class DefaultSchedulerComponent implements SchedulerComponent{ + + protected ScheduledExecutorService schedulerExecutor; + + @PostConstruct + public void init(){ + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("queue-scheduler")); + } + + @PreDestroy + public void destroy() { + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + } + + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(command, delay, unit); + } + + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(callable, delay, unit); + } + + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { + return schedulerExecutor.scheduleAtFixedRate(command, initialDelay, period, unit); + } + + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { + return schedulerExecutor.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java new file mode 100644 index 0000000000..3c12be43a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public interface SchedulerComponent { + + ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit); + + ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit); + + ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit); + + ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 1c260c754d..d406139dbb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -26,6 +26,9 @@ public class TbQueueCoreSettings { @Value("${queue.core.topic}") private String topic; + @Value("${queue.core.usage-stats-topic}") + private String usageStatsTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java new file mode 100644 index 0000000000..0702322838 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; + +import javax.annotation.PostConstruct; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@Component +@Slf4j +public class DefaultTbApiUsageClient implements TbApiUsageClient { + + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + @Value("${usage.stats.report.interval:10}") + private int interval; + + private final ConcurrentMap[] values = new ConcurrentMap[ApiUsageRecordKey.values().length]; + private final PartitionService partitionService; + private final SchedulerComponent scheduler; + private final TbQueueProducerProvider producerProvider; + private TbQueueProducer> msgProducer; + + public DefaultTbApiUsageClient(PartitionService partitionService, SchedulerComponent scheduler, TbQueueProducerProvider producerProvider) { + this.partitionService = partitionService; + this.scheduler = scheduler; + this.producerProvider = producerProvider; + } + + @PostConstruct + private void init() { + if (enabled) { + msgProducer = this.producerProvider.getTbUsageStatsMsgProducer(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + values[key.ordinal()] = new ConcurrentHashMap<>(); + } + scheduler.scheduleWithFixedDelay(this::reportStats, new Random().nextInt(interval), interval, TimeUnit.SECONDS); + } + } + + private void reportStats() { + try { + ConcurrentMap report = new ConcurrentHashMap<>(); + + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + values[key.ordinal()].forEach(((tenantId, atomicLong) -> { + long value = atomicLong.getAndSet(0); + if (value > 0) { + ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> { + ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder(); + msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + return msg; + }); + msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build()); + } + })); + } + + report.forEach(((tenantId, builder) -> { + //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).newByTopic(msgProducer.getDefaultTopic()); + msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); + })); + log.info("Report statistics for: {} tenants", report.size()); + } catch (Exception e) { + log.warn("Failed to report statistics: ", e); + } + } + + @Override + public void report(TenantId tenantId, ApiUsageRecordKey key, long value) { + if (enabled) { + ConcurrentMap map = values[key.ordinal()]; + AtomicLong atomicValue = map.computeIfAbsent(tenantId, id -> new AtomicLong()); + atomicValue.addAndGet(value); + } + } + + @Override + public void report(TenantId tenantId, ApiUsageRecordKey key) { + report(tenantId, key, 1L); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java new file mode 100644 index 0000000000..31d85b27c7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; + +public interface TbApiUsageClient { + + void report(TenantId tenantId, ApiUsageRecordKey key, long value); + + void report(TenantId tenantId, ApiUsageRecordKey key); + +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 01d8ca4524..4b5a6028e6 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -186,6 +186,7 @@ message GetEntityProfileRequestMsg { message GetEntityProfileResponseMsg { string entityType = 1; bytes data = 2; + bytes apiState = 3; } message EntityUpdateMsg { @@ -532,3 +533,16 @@ message ToTransportMsg { EntityDeleteMsg entityDeleteMsg = 9; ProvisionDeviceResponseMsg provisionResponse = 10; } + +message UsageStatsKVProto{ + string key = 1; + int64 value = 2; +} + +message ToUsageStatsServiceMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; + repeated UsageStatsKVProto values = 5; +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 3723b19f6d..14712a7cb3 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; @@ -57,6 +58,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.session.DeviceSessionCtx; import org.thingsboard.server.transport.mqtt.session.GatewaySessionHandler; @@ -97,6 +99,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private final UUID sessionId; private final MqttTransportContext context; private final TransportService transportService; + private final SchedulerComponent scheduler; private final SslHandler sslHandler; private final ConcurrentMap mqttQoSMap; @@ -108,6 +111,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.sessionId = UUID.randomUUID(); this.context = context; this.transportService = context.getTransportService(); + this.scheduler = context.getScheduler(); this.sslHandler = sslHandler; this.mqttQoSMap = new ConcurrentHashMap<>(); this.deviceSessionCtx = new DeviceSessionCtx(sessionId, mqttQoSMap, context); @@ -333,7 +337,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } else { deviceSessionCtx.getContext().getProtoMqttAdaptor().convertToPublish(deviceSessionCtx, provisionResponseMsg).ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); } - transportService.getSchedulerExecutor().schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); + scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); } catch (Exception e) { log.trace("[{}] Failed to convert device attributes response to MQTT msg", sessionId, e); } @@ -627,7 +631,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void onError(Throwable e) { - log.warn("[{}] Failed to submit session event", sessionId, e); + if (e instanceof TbRateLimitsException) { + log.trace("[{}] Failed to submit session event", sessionId, e); + } else { + log.warn("[{}] Failed to submit session event", sessionId, e); + } ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.close(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index 66809f9e61..4b7677dd69 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -44,6 +45,8 @@ public abstract class TransportContext { private TransportService transportService; @Autowired private TbServiceInfoProvider serviceInfoProvider; + @Autowired + private SchedulerComponent scheduler; @Getter private ExecutorService executor; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 8f57db3897..980aa33221 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -17,16 +17,13 @@ package org.thingsboard.server.common.transport; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.limits.TransportRateLimitType; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; @@ -42,14 +39,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCre import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import java.util.concurrent.ScheduledExecutorService; - /** * Created by ashvayka on 04.10.18. */ public interface TransportService { - GetEntityProfileResponseMsg getRoutingInfo(GetEntityProfileRequestMsg msg); + GetEntityProfileResponseMsg getEntityProfile(GetEntityProfileRequestMsg msg); void process(DeviceTransportType transportType, ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); @@ -66,12 +61,6 @@ public interface TransportService { void process(ProvisionDeviceRequestMsg msg, TransportServiceCallback callback); - void onProfileUpdate(DeviceProfile deviceProfile); - - boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); - - boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints, TransportRateLimitType... limits); - void process(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback); @@ -92,8 +81,6 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); - ScheduledExecutorService getSchedulerExecutor(); - void registerAsyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener); void registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java deleted file mode 100644 index 362502bf17..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.limits; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.msg.tools.TbRateLimits; - -@Slf4j -@Component -public class DefaultTransportRateLimitFactory implements TransportRateLimitFactory { - - private static final DummyTransportRateLimit ALWAYS_TRUE = new DummyTransportRateLimit(); - - @Override - public TransportRateLimit create(TransportRateLimitType type, Object configuration) { - if (!StringUtils.isEmpty(configuration)) { - try { - return new SimpleTransportRateLimit(new TbRateLimits(configuration.toString()), configuration.toString()); - } catch (Exception e) { - log.warn("[{}] Failed to init rate limit with configuration: {}", type, configuration, e); - return ALWAYS_TRUE; - } - } else { - return ALWAYS_TRUE; - } - } - - @Override - public TransportRateLimit createDefault(TransportRateLimitType type) { - return ALWAYS_TRUE; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java index 569f5446db..b6f160ad1e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java @@ -17,116 +17,163 @@ package org.thingsboard.server.common.transport.limits; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.queue.util.TbTransportComponent; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; +import java.util.function.Function; @Service @TbTransportComponent @Slf4j public class DefaultTransportRateLimitService implements TransportRateLimitService { - private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private final static DummyTransportRateLimit ALLOW = new DummyTransportRateLimit(); + private final ConcurrentMap tenantAllowed = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - private final TransportRateLimitFactory rateLimitFactory; private final TransportTenantProfileCache tenantProfileCache; - public DefaultTransportRateLimitService(TransportRateLimitFactory rateLimitFactory, TransportTenantProfileCache tenantProfileCache) { - this.rateLimitFactory = rateLimitFactory; + public DefaultTransportRateLimitService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @Override - public TransportRateLimitType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints, TransportRateLimitType... limits) { - TransportRateLimit[] tenantLimits = getTenantRateLimits(tenantId); - TransportRateLimit[] deviceLimits = getDeviceRateLimits(tenantId, deviceId); - for (TransportRateLimitType limitType : limits) { - TransportRateLimit rateLimit; - if (limitType.isTenantLevel()) { - rateLimit = tenantLimits[limitType.ordinal()]; - } else { - rateLimit = deviceLimits[limitType.ordinal()]; - } - if (!rateLimit.tryConsume(limitType.isMessageLevel() ? 1L : dataPoints)) { - return limitType; - } + public EntityType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints) { + if (!tenantAllowed.getOrDefault(tenantId, Boolean.TRUE)) { + return EntityType.TENANT; + } + if (!checkEntityRateLimit(dataPoints, getTenantRateLimits(tenantId))) { + return EntityType.TENANT; + } + if (!checkEntityRateLimit(dataPoints, getDeviceRateLimits(tenantId, deviceId))) { + return EntityType.DEVICE; } return null; } + private boolean checkEntityRateLimit(int dataPoints, EntityTransportRateLimits tenantLimits) { + if (dataPoints > 0) { + return tenantLimits.getTelemetryMsgRateLimit().tryConsume() && tenantLimits.getTelemetryDataPointsRateLimit().tryConsume(dataPoints); + } else { + return tenantLimits.getRegularMsgRateLimit().tryConsume(); + } + } + @Override public void update(TenantProfileUpdateResult update) { - TransportRateLimit[] newLimits = createTransportRateLimits(update.getProfile()); + EntityTransportRateLimits tenantRateLimitPrototype = createRateLimits(update.getProfile(), true); + EntityTransportRateLimits deviceRateLimitPrototype = createRateLimits(update.getProfile(), false); for (TenantId tenantId : update.getAffectedTenants()) { - mergeLimits(tenantId, newLimits); + mergeLimits(tenantId, tenantRateLimitPrototype, perTenantLimits::get, perTenantLimits::put); + tenantDevices.get(tenantId).forEach(deviceId -> { + mergeLimits(deviceId, deviceRateLimitPrototype, perDeviceLimits::get, perDeviceLimits::put); + }); } } @Override public void update(TenantId tenantId) { - mergeLimits(tenantId, fetchProfileAndInit(tenantId)); + EntityTransportRateLimits tenantRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), true); + EntityTransportRateLimits deviceRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), false); + mergeLimits(tenantId, tenantRateLimitPrototype, perTenantLimits::get, perTenantLimits::put); + tenantDevices.get(tenantId).forEach(deviceId -> { + mergeLimits(deviceId, deviceRateLimitPrototype, perDeviceLimits::get, perDeviceLimits::put); + }); } @Override public void remove(TenantId tenantId) { perTenantLimits.remove(tenantId); + tenantDevices.remove(tenantId); } @Override public void remove(DeviceId deviceId) { perDeviceLimits.remove(deviceId); + tenantDevices.values().forEach(set -> set.remove(deviceId)); } - private void mergeLimits(TenantId tenantId, TransportRateLimit[] newRateLimits) { - TransportRateLimit[] oldRateLimits = perTenantLimits.get(tenantId); + @Override + public void update(TenantId tenantId, boolean allowed) { + tenantAllowed.put(tenantId, allowed); + } + + private void mergeLimits(T deviceId, EntityTransportRateLimits newRateLimits, + Function getFunction, + BiConsumer putFunction) { + EntityTransportRateLimits oldRateLimits = getFunction.apply(deviceId); if (oldRateLimits == null) { - perTenantLimits.put(tenantId, newRateLimits); + putFunction.accept(deviceId, newRateLimits); } else { - for (int i = 0; i < TransportRateLimitType.values().length; i++) { - TransportRateLimit newLimit = newRateLimits[i]; - TransportRateLimit oldLimit = oldRateLimits[i]; - if (newLimit != null && (oldLimit == null || !oldLimit.getConfiguration().equals(newLimit.getConfiguration()))) { - oldRateLimits[i] = newLimit; - } + EntityTransportRateLimits updated = merge(oldRateLimits, newRateLimits); + if (updated != null) { + putFunction.accept(deviceId, updated); } } } - private TransportRateLimit[] fetchProfileAndInit(TenantId tenantId) { - return perTenantLimits.computeIfAbsent(tenantId, tmp -> createTransportRateLimits(tenantProfileCache.get(tenantId))); + private EntityTransportRateLimits merge(EntityTransportRateLimits oldRateLimits, EntityTransportRateLimits newRateLimits) { + boolean regularUpdate = !oldRateLimits.getRegularMsgRateLimit().getConfiguration().equals(newRateLimits.getRegularMsgRateLimit().getConfiguration()); + boolean telemetryMsgRateUpdate = !oldRateLimits.getTelemetryMsgRateLimit().getConfiguration().equals(newRateLimits.getTelemetryMsgRateLimit().getConfiguration()); + boolean telemetryDataPointUpdate = !oldRateLimits.getTelemetryDataPointsRateLimit().getConfiguration().equals(newRateLimits.getTelemetryDataPointsRateLimit().getConfiguration()); + if (regularUpdate || telemetryMsgRateUpdate || telemetryDataPointUpdate) { + return new EntityTransportRateLimits( + regularUpdate ? newLimit(newRateLimits.getRegularMsgRateLimit().getConfiguration()) : oldRateLimits.getRegularMsgRateLimit(), + telemetryMsgRateUpdate ? newLimit(newRateLimits.getTelemetryMsgRateLimit().getConfiguration()) : oldRateLimits.getTelemetryMsgRateLimit(), + telemetryDataPointUpdate ? newLimit(newRateLimits.getTelemetryDataPointsRateLimit().getConfiguration()) : oldRateLimits.getTelemetryDataPointsRateLimit()); + } else { + return null; + } } - private TransportRateLimit[] createTransportRateLimits(TenantProfile tenantProfile) { + private EntityTransportRateLimits createRateLimits(TenantProfile tenantProfile, boolean tenant) { TenantProfileData profileData = tenantProfile.getProfileData(); - TransportRateLimit[] rateLimits = new TransportRateLimit[TransportRateLimitType.values().length]; - for (TransportRateLimitType type : TransportRateLimitType.values()) { - rateLimits[type.ordinal()] = rateLimitFactory.create(type, profileData.getProperties().get(type.getConfigurationKey())); + DefaultTenantProfileConfiguration profile = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); + if (profile == null) { + return new EntityTransportRateLimits(ALLOW, ALLOW, ALLOW); + } else { + TransportRateLimit regularMsgRateLimit = newLimit(tenant ? profile.getTransportTenantMsgRateLimit() : profile.getTransportDeviceMsgRateLimit()); + TransportRateLimit telemetryMsgRateLimit = newLimit(tenant ? profile.getTransportTenantTelemetryMsgRateLimit() : profile.getTransportDeviceTelemetryMsgRateLimit()); + TransportRateLimit telemetryDpRateLimit = newLimit(tenant ? profile.getTransportTenantTelemetryDataPointsRateLimit() : profile.getTransportTenantTelemetryDataPointsRateLimit()); + return new EntityTransportRateLimits(regularMsgRateLimit, telemetryMsgRateLimit, telemetryDpRateLimit); } - return rateLimits; } - private TransportRateLimit[] getTenantRateLimits(TenantId tenantId) { - TransportRateLimit[] limits = perTenantLimits.get(tenantId); + private static TransportRateLimit newLimit(String config) { + return StringUtils.isEmpty(config) ? ALLOW : new SimpleTransportRateLimit(config); + } + + private EntityTransportRateLimits getTenantRateLimits(TenantId tenantId) { + EntityTransportRateLimits limits = perTenantLimits.get(tenantId); if (limits == null) { - limits = fetchProfileAndInit(tenantId); + limits = createRateLimits(tenantProfileCache.get(tenantId), true); perTenantLimits.put(tenantId, limits); } return limits; } - private TransportRateLimit[] getDeviceRateLimits(TenantId tenantId, DeviceId deviceId) { - TransportRateLimit[] limits = perDeviceLimits.get(deviceId); + private EntityTransportRateLimits getDeviceRateLimits(TenantId tenantId, DeviceId deviceId) { + EntityTransportRateLimits limits = perDeviceLimits.get(deviceId); if (limits == null) { - limits = fetchProfileAndInit(tenantId); + limits = createRateLimits(tenantProfileCache.get(tenantId), false); perDeviceLimits.put(deviceId, limits); + tenantDevices.computeIfAbsent(tenantId, id -> ConcurrentHashMap.newKeySet()).add(deviceId); } return limits; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java new file mode 100644 index 0000000000..50831ccff7 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.limits; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class EntityTransportRateLimits { + + private TransportRateLimit regularMsgRateLimit; + private TransportRateLimit telemetryMsgRateLimit; + private TransportRateLimit telemetryDataPointsRateLimit; + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java index 3253272ded..0ea920a4ba 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java @@ -26,6 +26,11 @@ public class SimpleTransportRateLimit implements TransportRateLimit { @Getter private final String configuration; + public SimpleTransportRateLimit(String configuration) { + this.configuration = configuration; + this.rateLimit = new TbRateLimits(configuration); + } + @Override public boolean tryConsume() { return rateLimit.tryConsume(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java index a97fbfc61d..3918927c0b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.common.transport.limits; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; public interface TransportRateLimitService { - TransportRateLimitType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints, TransportRateLimitType... limits); + EntityType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints); void update(TenantProfileUpdateResult update); @@ -31,4 +32,5 @@ public interface TransportRateLimitService { void remove(DeviceId deviceId); + void update(TenantId tenantId, boolean transportEnabled); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java deleted file mode 100644 index a3e6da6683..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.limits; - -import lombok.Getter; - -public enum TransportRateLimitType { - - TENANT_MAX_MSGS("transport.tenant.msg", true, true), - TENANT_TELEMETRY_MSGS("transport.tenant.telemetry", true, true), - TENANT_MAX_DATA_POINTS("transport.tenant.dataPoints", true, false), - DEVICE_MAX_MSGS("transport.device.msg", false, true), - DEVICE_TELEMETRY_MSGS("transport.device.telemetry", false, true), - DEVICE_MAX_DATA_POINTS("transport.device.dataPoints", false, false); - - @Getter - private final String configurationKey; - @Getter - private final boolean tenantLevel; - @Getter - private final boolean deviceLevel; - @Getter - private final boolean messageLevel; - @Getter - private final boolean dataPointLevel; - - TransportRateLimitType(String configurationKey, boolean tenantLevel, boolean messageLevel) { - this.configurationKey = configurationKey; - this.tenantLevel = tenantLevel; - this.deviceLevel = !tenantLevel; - this.messageLevel = messageLevel; - this.dataPointLevel = !messageLevel; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 089b8aaf53..4af723f3a0 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -25,6 +25,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; @@ -53,8 +55,6 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.limits.TransportRateLimitType; -import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; @@ -77,6 +77,8 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.queue.util.TbTransportComponent; import javax.annotation.PostConstruct; @@ -92,7 +94,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -122,8 +123,10 @@ public class DefaultTransportService implements TransportService { private final StatsFactory statsFactory; private final TransportDeviceProfileCache deviceProfileCache; private final TransportTenantProfileCache tenantProfileCache; + private final TbApiUsageClient apiUsageClient; private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; + private final SchedulerComponent scheduler; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -134,7 +137,6 @@ public class DefaultTransportService implements TransportService { protected MessagesStats tbCoreProducerStats; protected MessagesStats transportApiStats; - protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; private ExecutorService mainConsumerExecutor; @@ -150,7 +152,8 @@ public class DefaultTransportService implements TransportService { StatsFactory statsFactory, TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, - TransportRateLimitService rateLimitService, DataDecodingEncodingService dataDecodingEncodingService) { + TbApiUsageClient apiUsageClient, TransportRateLimitService rateLimitService, + DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; @@ -158,8 +161,10 @@ public class DefaultTransportService implements TransportService { this.statsFactory = statsFactory; this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; + this.apiUsageClient = apiUsageClient; this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; + this.scheduler = scheduler; } @PostConstruct @@ -167,9 +172,8 @@ public class DefaultTransportService implements TransportService { this.ruleEngineProducerStats = statsFactory.createMessagesStats(StatsType.RULE_ENGINE.getName() + ".producer"); this.tbCoreProducerStats = statsFactory.createMessagesStats(StatsType.CORE.getName() + ".producer"); this.transportApiStats = statsFactory.createMessagesStats(StatsType.TRANSPORT.getName() + ".producer"); - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + this.scheduler.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); transportApiRequestTemplate.setMessagesStats(transportApiStats); ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); @@ -215,9 +219,6 @@ public class DefaultTransportService implements TransportService { if (transportNotificationsConsumer != null) { transportNotificationsConsumer.unsubscribe(); } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } if (transportCallbackExecutor != null) { transportCallbackExecutor.shutdownNow(); } @@ -229,18 +230,13 @@ public class DefaultTransportService implements TransportService { } } - @Override - public ScheduledExecutorService getSchedulerExecutor() { - return this.schedulerExecutor; - } - @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } @Override - public TransportProtos.GetEntityProfileResponseMsg getRoutingInfo(TransportProtos.GetEntityProfileRequestMsg msg) { + public TransportProtos.GetEntityProfileResponseMsg getEntityProfile(TransportProtos.GetEntityProfileRequestMsg msg) { TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setEntityProfileRequestMsg(msg).build()); try { @@ -366,11 +362,11 @@ public class DefaultTransportService implements TransportService { for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { dataPoints += tsKv.getKvCount(); } - if (checkLimits(sessionInfo, msg, callback, dataPoints, TELEMETRY)) { + if (checkLimits(sessionInfo, msg, callback, dataPoints)) { reportActivityInternal(sessionInfo); TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), new ApiStatsProxyCallback<>(tenantId, dataPoints, callback)); for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("deviceName", sessionInfo.getDeviceName()); @@ -385,7 +381,7 @@ public class DefaultTransportService implements TransportService { @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback, msg.getKvCount(), TELEMETRY)) { + if (checkLimits(sessionInfo, msg, callback, msg.getKvCount())) { reportActivityInternal(sessionInfo); TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); @@ -479,7 +475,7 @@ public class DefaultTransportService implements TransportService { SessionMsgType.TO_SERVER_RPC_REQUEST, new TransportTbQueueCallback(callback)); String requestId = sessionId + "-" + msg.getRequestId(); toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); - schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); + scheduler.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); } } @@ -552,7 +548,7 @@ public class DefaultTransportService implements TransportService { SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + ScheduledFuture executorFuture = scheduler.schedule(() -> { listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); deregisterSession(sessionInfo); }, timeout, TimeUnit.MILLISECONDS); @@ -570,31 +566,23 @@ public class DefaultTransportService implements TransportService { sessions.remove(toSessionId(sessionInfo)); } - private TransportRateLimitType[] DEFAULT = new TransportRateLimitType[]{TransportRateLimitType.TENANT_MAX_MSGS, TransportRateLimitType.DEVICE_MAX_MSGS}; - private TransportRateLimitType[] TELEMETRY = TransportRateLimitType.values(); - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - return checkLimits(sessionInfo, msg, callback, 0, DEFAULT); + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + return checkLimits(sessionInfo, msg, callback, 0); } - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints, TransportRateLimitType... limits) { + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints) { if (log.isTraceEnabled()) { log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - TransportRateLimitType limit = rateLimitService.checkLimits(tenantId, deviceId, 0, limits); - if (limit == null) { + EntityType rateLimitedEntityType = rateLimitService.checkLimits(tenantId, deviceId, dataPoints); + if (rateLimitedEntityType == null) { return true; } else { if (callback != null) { - callback.onError(new TbRateLimitsException(limit.isTenantLevel() ? EntityType.TENANT : EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] {} rateLimit detected: {}", toSessionId(sessionInfo), tenantId, limit, msg); + callback.onError(new TbRateLimitsException(rateLimitedEntityType)); } return false; } @@ -637,8 +625,7 @@ public class DefaultTransportService implements TransportService { onProfileUpdate(deviceProfile); } } else if (EntityType.TENANT_PROFILE.equals(entityType)) { - TenantProfileUpdateResult update = tenantProfileCache.put(msg.getData()); - rateLimitService.update(update); + rateLimitService.update(tenantProfileCache.put(msg.getData())); } else if (EntityType.TENANT.equals(entityType)) { Optional profileOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); if (profileOpt.isPresent()) { @@ -648,6 +635,13 @@ public class DefaultTransportService implements TransportService { rateLimitService.update(tenant.getId()); } } + } else if (EntityType.API_USAGE_STATE.equals(entityType)) { + Optional stateOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); + if (stateOpt.isPresent()) { + ApiUsageState apiUsageState = stateOpt.get(); + rateLimitService.update(apiUsageState.getTenantId(), apiUsageState.isTransportEnabled()); + //TODO: if transport is disabled, we should close all sessions and not to check credentials. + } } } else if (toSessionMsg.hasEntityDeleteMsg()) { TransportProtos.EntityDeleteMsg msg = toSessionMsg.getEntityDeleteMsg(); @@ -669,8 +663,7 @@ public class DefaultTransportService implements TransportService { } } - @Override - public void onProfileUpdate(DeviceProfile deviceProfile) { + private void onProfileUpdate(DeviceProfile deviceProfile) { long deviceProfileIdMSB = deviceProfile.getId().getId().getMostSignificantBits(); long deviceProfileIdLSB = deviceProfile.getId().getId().getLeastSignificantBits(); sessions.forEach((id, md) -> { @@ -815,4 +808,31 @@ public class DefaultTransportService implements TransportService { callback.onError(t); } } + + private class ApiStatsProxyCallback implements TransportServiceCallback { + private final TenantId tenantId; + private final int dataPoints; + private final TransportServiceCallback callback; + + public ApiStatsProxyCallback(TenantId tenantId, int dataPoints, TransportServiceCallback callback) { + this.tenantId = tenantId; + this.dataPoints = dataPoints; + this.callback = callback; + } + + @Override + public void onSuccess(T msg) { + try { + apiUsageClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_MSG_COUNT, 1); + apiUsageClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_DP_COUNT, dataPoints); + } finally { + callback.onSuccess(msg); + } + } + + @Override + public void onError(Throwable e) { + callback.onError(e); + } + } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 784f7b3ae8..6773d09417 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -18,21 +18,19 @@ package org.thingsboard.server.common.transport.service; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; +import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.util.TbTransportComponent; import java.util.Collections; @@ -54,8 +52,15 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil private final ConcurrentMap> tenantProfileIds = new ConcurrentHashMap<>(); private final DataDecodingEncodingService dataDecodingEncodingService; + private TransportRateLimitService rateLimitService; private TransportService transportService; + @Lazy + @Autowired + public void setRateLimitService(TransportRateLimitService rateLimitService) { + this.rateLimitService = rateLimitService; + } + @Lazy @Autowired public void setTransportService(TransportService transportService) { @@ -77,7 +82,8 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil if (profileOpt.isPresent()) { TenantProfile newProfile = profileOpt.get(); log.trace("[{}] put: {}", newProfile.getId(), newProfile); - return new TenantProfileUpdateResult(newProfile, tenantProfileIds.get(newProfile.getId())); + Set affectedTenants = tenantProfileIds.get(newProfile.getId()); + return new TenantProfileUpdateResult(newProfile, affectedTenants != null ? affectedTenants : Collections.emptySet()); } else { log.warn("Failed to decode profile: {}", profileBody.toString()); return new TenantProfileUpdateResult(null, Collections.emptySet()); @@ -127,8 +133,8 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil .setEntityIdMSB(tenantId.getId().getMostSignificantBits()) .setEntityIdLSB(tenantId.getId().getLeastSignificantBits()) .build(); - TransportProtos.GetEntityProfileResponseMsg routingInfo = transportService.getRoutingInfo(msg); - Optional profileOpt = dataDecodingEncodingService.decode(routingInfo.getData().toByteArray()); + TransportProtos.GetEntityProfileResponseMsg entityProfileMsg = transportService.getEntityProfile(msg); + Optional profileOpt = dataDecodingEncodingService.decode(entityProfileMsg.getData().toByteArray()); if (profileOpt.isPresent()) { profile = profileOpt.get(); TenantProfile existingProfile = profiles.get(profile.getId()); @@ -140,9 +146,11 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil tenantProfileIds.computeIfAbsent(profile.getId(), id -> ConcurrentHashMap.newKeySet()).add(tenantId); tenantIds.put(tenantId, profile.getId()); } else { - log.warn("[{}] Can't decode tenant profile: {}", tenantId, routingInfo.getData()); + log.warn("[{}] Can't decode tenant profile: {}", tenantId, entityProfileMsg.getData()); throw new RuntimeException("Can't decode tenant profile!"); } + Optional apiStateOpt = dataDecodingEncodingService.decode(entityProfileMsg.getApiState().toByteArray()); + apiStateOpt.ifPresent(apiUsageState -> rateLimitService.update(tenantId, apiUsageState.isTransportEnabled())); } } finally { tenantProfileFetchLock.unlock(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f7e8711dcb..8e1e62b90f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -439,6 +439,18 @@ public class ModelConstants { public static final String OAUTH2_TEMPLATE_LOGIN_BUTTON_LABEL_PROPERTY = OAUTH2_LOGIN_BUTTON_LABEL_PROPERTY; public static final String OAUTH2_TEMPLATE_HELP_LINK_PROPERTY = "help_link"; + /** + * Usage Record constants. + */ + public static final String API_USAGE_STATE_TABLE_NAME = "api_usage_state"; + public static final String API_USAGE_STATE_TENANT_ID_COLUMN = TENANT_ID_PROPERTY; + public static final String API_USAGE_STATE_ENTITY_TYPE_COLUMN = ENTITY_TYPE_COLUMN; + public static final String API_USAGE_STATE_ENTITY_ID_COLUMN = ENTITY_ID_COLUMN; + public static final String API_USAGE_STATE_TRANSPORT_ENABLED_COLUMN = "transport_enabled"; + public static final String API_USAGE_STATE_DB_STORAGE_ENABLED_COLUMN = "db_storage_enabled"; + public static final String API_USAGE_STATE_RE_EXEC_ENABLED_COLUMN = "re_exec_enabled"; + public static final String API_USAGE_STATE_JS_EXEC_ENABLED_COLUMN = "js_exec_enabled"; + /** * Cassandra attributes and timeseries constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java new file mode 100644 index 0000000000..262252517a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -0,0 +1,103 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.TypeDef; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.util.mapping.JsonStringType; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import java.util.UUID; + +/** + * Created by Valerii Sosliuk on 4/21/2017. + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@TypeDef(name = "json", typeClass = JsonStringType.class) +@Table(name = ModelConstants.API_USAGE_STATE_TABLE_NAME) +public class ApiUsageStateEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = ModelConstants.API_USAGE_STATE_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = ModelConstants.API_USAGE_STATE_ENTITY_TYPE_COLUMN) + private String entityType; + + @Column(name = ModelConstants.API_USAGE_STATE_ENTITY_ID_COLUMN) + private UUID entityId; + + @Column(name = ModelConstants.API_USAGE_STATE_TRANSPORT_ENABLED_COLUMN) + private boolean transportEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_DB_STORAGE_ENABLED_COLUMN) + private boolean dbStorageEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_RE_EXEC_ENABLED_COLUMN) + private boolean reExecEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_JS_EXEC_ENABLED_COLUMN) + private boolean jsExecEnabled = true; + + public ApiUsageStateEntity() { + } + + public ApiUsageStateEntity(ApiUsageState ur) { + if (ur.getId() != null) { + this.setUuid(ur.getId().getId()); + } + this.setCreatedTime(ur.getCreatedTime()); + if (ur.getTenantId() != null) { + this.tenantId = ur.getTenantId().getId(); + } + if (ur.getEntityId() != null) { + this.entityType = ur.getEntityId().getEntityType().name(); + this.entityId = ur.getEntityId().getId(); + } + this.transportEnabled = ur.isTransportEnabled(); + this.dbStorageEnabled = ur.isDbStorageEnabled(); + this.reExecEnabled = ur.isReExecEnabled(); + this.jsExecEnabled = ur.isJsExecEnabled(); + } + + @Override + public ApiUsageState toData() { + ApiUsageState ur = new ApiUsageState(new ApiUsageStateId(this.getUuid())); + ur.setCreatedTime(createdTime); + if (tenantId != null) { + ur.setTenantId(new TenantId(tenantId)); + } + if (entityId != null) { + ur.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + ur.setTransportEnabled(transportEnabled); + ur.setDbStorageEnabled(dbStorageEnabled); + ur.setReExecEnabled(reExecEnabled); + ur.setJsExecEnabled(jsExecEnabled); + return ur; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java index 7c69f58935..7663682496 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java @@ -22,14 +22,13 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JsonBinaryType; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java index 1d7e8af03f..7bc77e14a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java @@ -79,6 +79,20 @@ public class BaseRuleNodeStateService extends AbstractEntityService implements R ruleNodeStateDao.removeByRuleNodeId(ruleNodeId.getId()); } + @Override + public void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId ruleNodeId, EntityId entityId) { + if (tenantId == null) { + throw new DataValidationException("Tenant id should be specified!."); + } + if (ruleNodeId == null) { + throw new DataValidationException("Rule node id should be specified!."); + } + if (entityId == null) { + throw new DataValidationException("Entity id should be specified!."); + } + ruleNodeStateDao.removeByRuleNodeIdAndEntityId(ruleNodeId.getId(), entityId.getId()); + } + public RuleNodeState saveOrUpdate(TenantId tenantId, RuleNodeState ruleNodeState, boolean update) { try { if (update) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java index 89c2baea7f..9f304dd828 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java @@ -35,4 +35,6 @@ public interface RuleNodeStateDao extends Dao { RuleNodeState findByRuleNodeIdAndEntityId(UUID ruleNodeId, UUID entityId); void removeByRuleNodeId(UUID ruleNodeId); + + void removeByRuleNodeIdAndEntityId(UUID ruleNodeId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 75f7658a5a..380e166aa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -210,6 +210,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.CUSTOMER, "customer"); entityTableMap.put(EntityType.USER, "tb_user"); entityTableMap.put(EntityType.TENANT, "tenant"); + entityTableMap.put(EntityType.API_USAGE_STATE, "(select aus.id, aus.created_time, aus.tenant_id, '' as name, '' as additional_info from api_usage_state as aus)"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ @@ -431,6 +432,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { case DEVICE_SEARCH_QUERY: case ASSET_SEARCH_QUERY: case ENTITY_VIEW_SEARCH_QUERY: + case API_USAGE_STATE: return ""; default: throw new RuntimeException("Not implemented!"); @@ -682,6 +684,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return EntityType.ENTITY_VIEW; case RELATIONS_QUERY: return ((RelationsQueryFilter) entityFilter).getRootEntity().getEntityType(); + case API_USAGE_STATE: + return EntityType.API_USAGE_STATE; default: throw new RuntimeException("Not implemented!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java index c61e58353c..e32f90a645 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java @@ -63,4 +63,10 @@ public class JpaRuleNodeStateDao extends JpaAbstractDao { + + @Query("SELECT ur FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId " + + "AND ur.entityId = :tenantId AND ur.entityType = 'TENANT' ") + ApiUsageStateEntity findByTenantId(@Param("tenantId") UUID tenantId); + + @Transactional + @Modifying + @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") + void deleteApiUsageStateByTenantId(@Param("tenantId") UUID tenantId); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java new file mode 100644 index 0000000000..2c6b2e924d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.usagerecord; + +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; + +import java.util.UUID; + +/** + * @author Andrii Shvaika + */ +@Component +public class JpaApiUsageStateDao extends JpaAbstractDao implements ApiUsageStateDao { + + private final ApiUsageStateRepository apiUsageStateRepository; + + public JpaApiUsageStateDao(ApiUsageStateRepository apiUsageStateRepository) { + this.apiUsageStateRepository = apiUsageStateRepository; + } + + @Override + protected Class getEntityClass() { + return ApiUsageStateEntity.class; + } + + @Override + protected CrudRepository getCrudRepository() { + return apiUsageStateRepository; + } + + @Override + public ApiUsageState findTenantApiUsageState(UUID tenantId) { + return DaoUtil.getData(apiUsageStateRepository.findByTenantId(tenantId)); + } + + @Override + public void deleteApiUsageStateByTenantId(TenantId tenantId) { + apiUsageStateRepository.deleteApiUsageStateByTenantId(tenantId.getId()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 3cee730ee3..616260ba45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -100,7 +100,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { return Futures.immediateFuture(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 8dcefeca61..c585804b82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -30,11 +30,14 @@ import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { + protected static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + @Autowired protected ScheduledLogExecutorComponent logExecutor; @@ -56,6 +59,9 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries @Value("${sql.batch_sort:false}") protected boolean batchSortEnabled; + @Value("${sql.ttl.ts.ts_key_value_ttl:0}") + private long systemTtl; + protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() @@ -75,4 +81,19 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries } }, service); } + + protected long computeTtl(long ttl) { + if (systemTtl > 0) { + if (ttl == 0) { + ttl = systemTtl; + } else { + ttl = Math.min(systemTtl, ttl); + } + } + return ttl; + } + + protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { + return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java index 5ea387328d..025134e187 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.hsql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; @@ -35,7 +37,8 @@ import org.thingsboard.server.dao.util.SqlTsDao; public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao implements TimeseriesDao { @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TsKvEntity entity = new TsKvEntity(); @@ -48,7 +51,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 8c50bcb38c..3d971f61c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.psql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -71,7 +73,8 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); savePartitionIfNotExist(tsKvEntry.getTs()); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); @@ -85,7 +88,7 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } private void savePartitionIfNotExist(long ts) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 6d5923c508..f6fb027d7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -101,7 +102,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); @@ -113,14 +115,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(0); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileDao.java index 1c9e50431f..497c21720d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileDao.java @@ -37,4 +37,5 @@ public interface TenantProfileDao extends Dao { TenantProfile findDefaultTenantProfile(TenantId tenantId); EntityInfo findDefaultTenantProfileInfo(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 939e2b7426..95dd0ab45a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -23,20 +23,19 @@ import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.util.mapping.JacksonUtil; import java.util.Arrays; import java.util.Collections; @@ -147,7 +146,9 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T defaultTenantProfile = new TenantProfile(); defaultTenantProfile.setDefault(true); defaultTenantProfile.setName("Default"); - defaultTenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData profileData = new TenantProfileData(); + profileData.setConfiguration(new DefaultTenantProfileConfiguration()); + defaultTenantProfile.setProfileData(profileData); defaultTenantProfile.setDescription("Default tenant profile"); defaultTenantProfile.setIsolatedTbCore(false); defaultTenantProfile.setIsolatedTbRuleEngine(false); @@ -215,6 +216,12 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T if (StringUtils.isEmpty(tenantProfile.getName())) { throw new DataValidationException("Tenant profile name should be specified!"); } + if (tenantProfile.getProfileData() == null) { + throw new DataValidationException("Tenant profile data should be specified!"); + } + if (tenantProfile.getProfileData().getConfiguration() == null) { + throw new DataValidationException("Tenant profile data configuration should be specified!"); + } if (tenantProfile.isDefault()) { TenantProfile defaultTenantProfile = findDefaultTenantProfile(tenantId); if (defaultTenantProfile != null && !defaultTenantProfile.getId().equals(tenantProfile.getId())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 52761184fd..3a0edf5eea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -72,6 +73,9 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private DeviceProfileService deviceProfileService; + @Autowired + private ApiUsageStateService apiUsageStateService; + @Autowired private EntityViewService entityViewService; @@ -117,6 +121,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe Tenant savedTenant = tenantDao.save(tenant.getId(), tenant); if (tenant.getId() == null) { deviceProfileService.createDefaultDeviceProfile(savedTenant.getId()); + apiUsageStateService.createDefaultApiUsageState(savedTenant.getId()); } return savedTenant; } @@ -134,6 +139,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); userService.deleteTenantAdmins(tenantId); ruleChainService.deleteRuleChainsByTenantId(tenantId); + apiUsageStateService.deleteApiUsageStateByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); deleteEntityRelations(tenantId, tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index b86d6128f6..3b97e2bb82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.timeseries; +import com.google.common.base.Function; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -55,6 +57,20 @@ public class BaseTimeseriesService implements TimeseriesService { private static final int INSERTS_PER_ENTRY = 3; private static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY; + public static final Function, Integer> SUM_ALL_INTEGERS = new Function, Integer>() { + @Override + public @Nullable Integer apply(@Nullable List input) { + int result = 0; + if (input != null) { + for (Integer tmp : input) { + if (tmp != null) { + result += tmp; + } + } + } + return result; + } + }; @Value("${database.ts_max_intervals}") private long maxTsIntervals; @@ -101,26 +117,26 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } - List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); + List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); for (TsKvEntry tsKvEntry : tsKvEntries) { if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); } - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override @@ -135,12 +151,12 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.allAsList(futures); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey(), ttl)); - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } @@ -216,7 +232,7 @@ public class BaseTimeseriesService implements TimeseriesService { } else if (query.getAggregation() == null) { throw new IncorrectParameterException("Incorrect ReadTsKvQuery. Aggregation can't be empty"); } - if(!Aggregation.NONE.equals(query.getAggregation())) { + if (!Aggregation.NONE.equals(query.getAggregation())) { long step = Math.max(query.getInterval(), 1000); long intervalCounts = (query.getEndTs() - query.getStartTs()) / step; if (intervalCounts > maxTsIntervals || intervalCounts < 0) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 2f3fe4854c..b9d8c62833 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -60,6 +60,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -74,6 +75,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; + public static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); @Autowired @@ -141,9 +144,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { List> futures = new ArrayList<>(); ttl = computeTtl(ttl); + int dataPointDays = tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); long partition = toPartitionTs(tsKvEntry.getTs()); DataType type = tsKvEntry.getDataType(); if (setNullValuesEnabled) { @@ -161,11 +165,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } BoundStatement stmt = stmtBuilder.build(); futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + return Futures.transform(Futures.allAsList(futures), result -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { if (isFixedPartitioning()) { return Futures.immediateFuture(null); } @@ -181,7 +185,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD stmtBuilder.setInt(4, (int) ttl); } BoundStatement stmt = stmtBuilder.build(); - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> 0); } @Override @@ -649,9 +653,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } /** - // * Select existing partitions from the table - // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity - // */ + * // * Select existing partitions from the table + * // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity + * // + */ private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index dda2b227c3..2dd52580b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -31,9 +31,9 @@ public interface TimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); - ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); + ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java new file mode 100644 index 0000000000..1521da525e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; + +import java.util.UUID; + +public interface ApiUsageStateDao extends Dao { + + /** + * Save or update usage record object + * + * @param apiUsageState the usage record + * @return saved usage record entity + */ + ApiUsageState save(TenantId tenantId, ApiUsageState apiUsageState); + + /** + * Find usage record by tenantId. + * + * @param tenantId the tenantId + * @return the corresponding usage record + */ + ApiUsageState findTenantApiUsageState(UUID tenantId); + + /** + * Delete usage record by tenantId. + * + * @param tenantId the tenantId + */ + void deleteApiUsageStateByTenantId(TenantId tenantId); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java new file mode 100644 index 0000000000..e1b494e2e8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantProfileDao; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service +@Slf4j +public class ApiUsageStateServiceImpl extends AbstractEntityService implements ApiUsageStateService { + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + + private final ApiUsageStateDao apiUsageStateDao; + private final TenantProfileDao tenantProfileDao; + private final TenantDao tenantDao; + private final TimeseriesService tsService; + + public ApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, TimeseriesService tsService) { + this.tenantDao = tenantDao; + this.apiUsageStateDao = apiUsageStateDao; + this.tenantProfileDao = tenantProfileDao; + this.tsService = tsService; + } + + @Override + public void deleteApiUsageStateByTenantId(TenantId tenantId) { + log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + apiUsageStateDao.deleteApiUsageStateByTenantId(tenantId); + } + + @Override + public ApiUsageState createDefaultApiUsageState(TenantId tenantId) { + log.trace("Executing createDefaultUsageRecord [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(tenantId); + apiUsageStateValidator.validate(apiUsageState, ApiUsageState::getTenantId); + + ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + + Tenant tenant = tenantDao.findById(tenantId, tenantId.getId()); + TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenant.getTenantProfileId().getId()); + TenantProfileConfiguration configuration = tenantProfile.getProfileData().getConfiguration(); + List profileThresholds = new ArrayList<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + profileThresholds.add(new BasicTsKvEntry(saved.getCreatedTime(), new LongDataEntry(key.getApiLimitKey(), configuration.getProfileThreshold(key)))); + } + tsService.save(tenantId, saved.getId(), profileThresholds, 0L); + return saved; + } + + @Override + public ApiUsageState update(ApiUsageState apiUsageState) { + log.trace("Executing save [{}]", apiUsageState.getTenantId()); + validateId(apiUsageState.getTenantId(), INCORRECT_TENANT_ID + apiUsageState.getTenantId()); + validateId(apiUsageState.getId(), "Can't save new usage state. Only update is allowed!"); + return apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + } + + @Override + public ApiUsageState findTenantApiUsageState(TenantId tenantId) { + log.trace("Executing findTenantUsageRecord, tenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + return apiUsageStateDao.findTenantApiUsageState(tenantId.getId()); + } + + @Override + public ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id) { + log.trace("Executing findApiUsageStateById, tenantId [{}], apiUsageStateId [{}]", tenantId, id); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(id, "Incorrect apiUsageStateId " + id); + return apiUsageStateDao.findById(tenantId, id.getId()); + } + + private DataValidator apiUsageStateValidator = + new DataValidator() { + @Override + protected void validateDataImpl(TenantId requestTenantId, ApiUsageState apiUsageState) { + if (apiUsageState.getTenantId() == null) { + throw new DataValidationException("ApiUsageState should be assigned to tenant!"); + } else { + Tenant tenant = tenantDao.findById(requestTenantId, apiUsageState.getTenantId().getId()); + if (tenant == null) { + throw new DataValidationException("Asset is referencing to non-existent tenant!"); + } + } + if (apiUsageState.getEntityId() == null) { + throw new DataValidationException("UsageRecord should be assigned to entity!"); + } else if (!EntityType.TENANT.equals(apiUsageState.getEntityId().getEntityType())) { + throw new DataValidationException("Only Tenant Usage Records are supported!"); + } else if (!apiUsageState.getTenantId().getId().equals(apiUsageState.getEntityId().getId())) { + throw new DataValidationException("Can't assign one Usage Record to multiple tenants!"); + } + } + }; + +} diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 28abe32786..b7c028d5d7 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -405,3 +405,16 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( help_link varchar(255), CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); + +CREATE TABLE IF NOT EXISTS api_usage_state ( + id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + entity_type varchar(32), + entity_id uuid, + transport_enabled boolean, + db_storage_enabled boolean, + re_exec_enabled boolean, + js_exec_enabled boolean, + CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) +); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 233cd509da..6fffea8fe9 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -432,6 +432,19 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); +CREATE TABLE IF NOT EXISTS api_usage_state ( + id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + entity_type varchar(32), + entity_id uuid, + transport_enabled boolean, + db_storage_enabled boolean, + re_exec_enabled boolean, + js_exec_enabled boolean, + CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) +); + CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index cf6f574180..2a99e868a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -27,7 +27,6 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; @@ -38,10 +37,7 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTra import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; @@ -61,6 +57,7 @@ import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -85,6 +82,9 @@ public abstract class AbstractServiceTest { @Autowired protected UserService userService; + @Autowired + protected ApiUsageStateService apiUsageStateService; + @Autowired protected AdminSettingsService adminSettingsService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java new file mode 100644 index 0000000000..a33e190a03 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; + + +public abstract class BaseApiUsageStateServiceTest extends AbstractServiceTest { + + private TenantId tenantId; + + @Before + public void before() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + Assert.assertNotNull(savedTenant); + tenantId = savedTenant.getId(); + } + + @After + public void after() { + tenantService.deleteTenant(tenantId); + } + + @Test + public void testFindApiUsageStateByTenantId() { + ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); + } + + @Test + public void testUpdateApiUsageState(){ + ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); + Assert.assertTrue(apiUsageState.isTransportEnabled()); + apiUsageState.setTransportEnabled(false); + apiUsageState = apiUsageStateService.update(apiUsageState); + Assert.assertNotNull(apiUsageState); + apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); + Assert.assertFalse(apiUsageState.isTransportEnabled()); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index d04d18772d..ff87e17d56 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -793,7 +793,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -1272,7 +1272,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index a9917126df..f93fc2fb1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -21,13 +21,13 @@ import org.junit.Test; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.util.mapping.JacksonUtil; import java.util.ArrayList; import java.util.Collections; @@ -188,8 +188,8 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { Assert.assertTrue(pageData.getData().isEmpty()); tenantProfiles.addAll(pageData.getData()); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile)); } @@ -224,8 +224,8 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { List tenantProfiles = new ArrayList<>(); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile)); } @@ -263,7 +263,9 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName(name); tenantProfile.setDescription(name + " Test"); - tenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData profileData = new TenantProfileData(); + profileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(profileData); tenantProfile.setDefault(false); tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbRuleEngine(false); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java new file mode 100644 index 0000000000..bbaae0a5a9 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.sql; + +import org.thingsboard.server.dao.service.BaseApiUsageStateServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@DaoSqlTest +public class ApiUsageStateServiceSqlTest extends BaseApiUsageStateServiceTest { +} diff --git a/dao/src/test/resources/sql/hsql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql index 160049a144..b5629b0560 100644 --- a/dao/src/test/resources/sql/hsql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/hsql/drop-all-tables.sql @@ -27,4 +27,5 @@ DROP TABLE IF EXISTS rule_chain; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS api_usage_state; DROP FUNCTION IF EXISTS to_uuid; diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index e277c39403..34985883d7 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -28,3 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS api_usage_state; \ No newline at end of file diff --git a/dao/src/test/resources/sql/system-test.sql b/dao/src/test/resources/sql/system-test.sql index e59afd914c..68a0e0efa3 100644 --- a/dao/src/test/resources/sql/system-test.sql +++ b/dao/src/test/resources/sql/system-test.sql @@ -1,2 +1,6 @@ +TRUNCATE TABLE device_credentials; +TRUNCATE TABLE device; +TRUNCATE TABLE device_profile; +TRUNCATE TABLE rule_node_state; TRUNCATE TABLE rule_node; TRUNCATE TABLE rule_chain; \ No newline at end of file diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index da7eaff876..211cebb2ad 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -28,3 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS api_usage_state; \ No newline at end of file diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceProfileCache.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceProfileCache.java index 53e2c558f6..b8e291ca74 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceProfileCache.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceProfileCache.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -32,7 +33,7 @@ public interface RuleEngineDeviceProfileCache { DeviceProfile get(TenantId tenantId, DeviceId deviceId); - void addListener(TenantId tenantId, EntityId listenerId, Consumer listener); + void addListener(TenantId tenantId, EntityId listenerId, Consumer profileListener, BiConsumer devicelistener); void removeListener(TenantId tenantId, EntityId listenerId); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index f48151f498..081a84eef0 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -47,6 +48,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -221,11 +223,13 @@ public interface TbContext { RuleNodeState findRuleNodeStateForEntity(EntityId entityId); + void removeRuleNodeStateForEntity(EntityId entityId); + RuleNodeState saveRuleNodeState(RuleNodeState state); void clearRuleNodeStates(); - void addProfileListener(Consumer listener); + void addDeviceProfileListeners(Consumer listener, BiConsumer deviceListener); void removeProfileListener(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 8be2b81a59..b1eeb82b15 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -99,6 +99,10 @@ class AlarmRuleState { } } for (EntityKey key : changedKeys) { + EntityKeyType keyType = key.getType(); + if (EntityKeyType.CLIENT_ATTRIBUTE.equals(keyType) || EntityKeyType.SERVER_ATTRIBUTE.equals(keyType) || EntityKeyType.SHARED_ATTRIBUTE.equals(keyType)) { + key = new EntityKey(EntityKeyType.ATTRIBUTE, key.getKey()); + } if (entityKeys.contains(key)) { return true; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java index df46a4bf88..6c771f0c86 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java @@ -103,6 +103,7 @@ class AlarmState { } AlarmEvalResult evalResult = evalFunction.apply(clearState, data); if (AlarmEvalResult.TRUE.equals(evalResult)) { + clearState.clear(); stateUpdate |= clearState.checkUpdate(); for (AlarmRuleState state : createRulesSortedBySeverityDesc) { state.clear(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java index b4bbf08323..fa7538bce6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java @@ -15,6 +15,8 @@ */ package org.thingsboard.rule.engine.profile; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.rule.engine.api.RuleNode; @@ -59,6 +61,7 @@ import java.util.concurrent.TimeUnit; public class TbDeviceProfileNode implements TbNode { private static final String PERIODIC_MSG_TYPE = "TbDeviceProfilePeriodicMsg"; private static final String PROFILE_UPDATE_MSG_TYPE = "TbDeviceProfileUpdateMsg"; + private static final String DEVICE_UPDATE_MSG_TYPE = "TbDeviceUpdateMsg"; private TbDeviceProfileNodeConfiguration config; private RuleEngineDeviceProfileCache cache; @@ -71,7 +74,7 @@ public class TbDeviceProfileNode implements TbNode { this.cache = ctx.getDeviceProfileCache(); this.ctx = ctx; scheduleAlarmHarvesting(ctx); - ctx.addProfileListener(this::onProfileUpdate); + ctx.addDeviceProfileListeners(this::onProfileUpdate, this::onDeviceUpdate); if (config.isFetchAlarmRulesStateOnStart()) { log.info("[{}] Fetching alarm rule state", ctx.getSelfId()); int fetchCount = 0; @@ -108,13 +111,22 @@ public class TbDeviceProfileNode implements TbNode { harvestAlarms(ctx, System.currentTimeMillis()); } else if (msg.getType().equals(PROFILE_UPDATE_MSG_TYPE)) { updateProfile(ctx, new DeviceProfileId(UUID.fromString(msg.getData()))); + } else if (msg.getType().equals(DEVICE_UPDATE_MSG_TYPE)) { + JsonNode data = JacksonUtil.toJsonNode(msg.getData()); + DeviceId deviceId = new DeviceId(UUID.fromString(data.get("deviceId").asText())); + if (data.has("profileId")) { + invalidateDeviceProfileCache(deviceId, new DeviceProfileId(UUID.fromString(data.get("deviceProfileId").asText()))); + } else { + removeDeviceState(deviceId); + } + } else { if (EntityType.DEVICE.equals(originatorType)) { DeviceId deviceId = new DeviceId(msg.getOriginator().getId()); if (msg.getType().equals(DataConstants.ENTITY_UPDATED)) { invalidateDeviceProfileCache(deviceId, msg.getData()); } else if (msg.getType().equals(DataConstants.ENTITY_DELETED)) { - deviceStates.remove(deviceId); + removeDeviceState(deviceId); } else { DeviceState deviceState = getOrCreateDeviceState(ctx, deviceId, null); if (deviceState != null) { @@ -182,15 +194,39 @@ public class TbDeviceProfileNode implements TbNode { ctx.tellSelf(TbMsg.newMsg(PROFILE_UPDATE_MSG_TYPE, ctx.getTenantId(), TbMsgMetaData.EMPTY, profile.getId().getId().toString()), 0L); } + private void onDeviceUpdate(DeviceId deviceId, DeviceProfile deviceProfile) { + ObjectNode msgData = JacksonUtil.newObjectNode(); + msgData.put("deviceId", deviceId.getId().toString()); + if (deviceProfile != null) { + msgData.put("deviceProfileId", deviceProfile.getId().getId().toString()); + } + ctx.tellSelf(TbMsg.newMsg(DEVICE_UPDATE_MSG_TYPE, ctx.getTenantId(), TbMsgMetaData.EMPTY, JacksonUtil.toString(msgData)), 0L); + } + protected void invalidateDeviceProfileCache(DeviceId deviceId, String deviceJson) { DeviceState deviceState = deviceStates.get(deviceId); if (deviceState != null) { DeviceProfileId currentProfileId = deviceState.getProfileId(); Device device = JacksonUtil.fromString(deviceJson, Device.class); if (!currentProfileId.equals(device.getDeviceProfileId())) { - deviceStates.remove(deviceId); + removeDeviceState(deviceId); } } } + protected void invalidateDeviceProfileCache(DeviceId deviceId, DeviceProfileId deviceProfileId) { + DeviceState deviceState = deviceStates.get(deviceId); + if (deviceState != null) { + if (!deviceState.getProfileId().equals(deviceProfileId)) { + removeDeviceState(deviceId); + } + } + } + + private void removeDeviceState(DeviceId deviceId) { + DeviceState state = deviceStates.remove(deviceId); + if (config.isPersistAlarmRulesState() && (state != null || !config.isFetchAlarmRulesStateOnStart())) { + ctx.removeRuleNodeStateForEntity(deviceId); + } + } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 8562b52bb9..ae6bd7ebbd 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.UUID; import static org.mockito.Mockito.verify; @@ -139,7 +140,7 @@ public class TbDeviceProfileNodeTest { DeviceProfileAlarm dpa = new DeviceProfileAlarm(); dpa.setId("highTemperatureAlarmID"); dpa.setAlarmType("highTemperatureAlarm"); - dpa.setCreateRules(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)); + dpa.setCreateRules(new LinkedHashMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); KeyFilter lowTempFilter = new KeyFilter(); lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index a510735855..acd89e3c1b 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -753,6 +753,9 @@ export class EntityService { case AliasFilterType.entityViewType: result.entityFilter = deepClone(filter); return of(result); + case AliasFilterType.apiUsageState: + result.entityFilter = deepClone(filter); + return of(result); case AliasFilterType.relationsQuery: result.stateEntity = filter.rootStateEntity; let rootEntityType; diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index d2e7cb0a28..ec7ccea660 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -113,6 +113,8 @@ import { AlarmScheduleInfoComponent } from './profile/alarm/alarm-schedule-info. import { AlarmScheduleDialogComponent } from '@home/components/profile/alarm/alarm-schedule-dialog.component'; import { EditAlarmDetailsDialogComponent } from './profile/alarm/edit-alarm-details-dialog.component'; import { AlarmRuleConditionDialogComponent } from '@home/components/profile/alarm/alarm-rule-condition-dialog.component'; +import { DefaultTenantProfileConfigurationComponent } from './profile/tenant/default-tenant-profile-configuration.component'; +import { TenantProfileConfigurationComponent } from './profile/tenant/tenant-profile-configuration.component'; @NgModule({ declarations: @@ -182,6 +184,8 @@ import { AlarmRuleConditionDialogComponent } from '@home/components/profile/alar FilterUserInfoDialogComponent, FilterPredicateValueComponent, TenantProfileAutocompleteComponent, + DefaultTenantProfileConfigurationComponent, + TenantProfileConfigurationComponent, TenantProfileDataComponent, TenantProfileComponent, TenantProfileDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts index 3cdb0b3a4e..9553e23528 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts @@ -204,7 +204,6 @@ export class TenantProfileAutocompleteComponent implements ControlValueAccessor, createTenantProfile($event: Event, profileName: string) { $event.preventDefault(); const tenantProfile: TenantProfile = { - id: null, name: profileName }; this.openTenantProfileDialog(tenantProfile, true); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html index a3d504d0e0..3d853940f4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html @@ -16,9 +16,15 @@ -->
- - + + + +
tenant-profile.profile-configuration
+ + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts index 377b6f0978..cc6ce7a6ce 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts @@ -62,7 +62,7 @@ export class TenantProfileDataComponent implements ControlValueAccessor, OnInit ngOnInit() { this.tenantProfileDataFormGroup = this.fb.group({ - tenantProfileData: [null, Validators.required] + configuration: [null, Validators.required] }); this.tenantProfileDataFormGroup.valueChanges.subscribe(() => { this.updateModel(); @@ -79,13 +79,13 @@ export class TenantProfileDataComponent implements ControlValueAccessor, OnInit } writeValue(value: TenantProfileData | null): void { - this.tenantProfileDataFormGroup.get('tenantProfileData').patchValue(value, {emitEvent: false}); + this.tenantProfileDataFormGroup.patchValue({configuration: value?.configuration}, {emitEvent: false}); } private updateModel() { let tenantProfileData: TenantProfileData = null; if (this.tenantProfileDataFormGroup.valid) { - tenantProfileData = this.tenantProfileDataFormGroup.getRawValue().tenantProfileData; + tenantProfileData = this.tenantProfileDataFormGroup.getRawValue(); } this.propagateChange(tenantProfileData); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 2aadb551de..7cca3ae414 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -18,7 +18,12 @@ import { Component, Inject, Input, Optional } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { TenantProfile } from '@app/shared/models/tenant.model'; +import { + createTenantProfileConfiguration, + TenantProfile, + TenantProfileData, + TenantProfileType +} from '@app/shared/models/tenant.model'; import { ActionNotificationShow } from '@app/core/notification/notification.actions'; import { TranslateService } from '@ngx-translate/core'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -56,7 +61,9 @@ export class TenantProfileComponent extends EntityComponent { name: [entity ? entity.name : '', [Validators.required]], isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], - profileData: [entity && !this.isAdd ? entity.profileData : {}, []], + profileData: [entity && !this.isAdd ? entity.profileData : { + configuration: createTenantProfileConfiguration(TenantProfileType.DEFAULT) + } as TenantProfileData, []], description: [entity ? entity.description : '', []], } ); @@ -66,7 +73,9 @@ export class TenantProfileComponent extends EntityComponent { this.entityForm.patchValue({name: entity.name}); this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); - this.entityForm.patchValue({profileData: entity.profileData}); + this.entityForm.patchValue({profileData: !this.isAdd ? entity.profileData: { + configuration: createTenantProfileConfiguration(TenantProfileType.DEFAULT) + } as TenantProfileData}); this.entityForm.patchValue({description: entity.description}); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html new file mode 100644 index 0000000000..cdb4febcc0 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -0,0 +1,141 @@ + +
+
+ + tenant-profile.maximum-devices + + + {{ 'tenant-profile.maximum-devices-required' | translate}} + + + {{ 'tenant-profile.maximum-devices-range' | translate}} + + + + tenant-profile.maximum-assets + + + {{ 'tenant-profile.maximum-assets-required' | translate}} + + + {{ 'tenant-profile.maximum-assets-range' | translate}} + + + + tenant-profile.max-transport-messages + + + {{ 'tenant-profile.max-transport-messages-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-required' | translate}} + + + + tenant-profile.max-transport-data-points + + + {{ 'tenant-profile.max-transport-data-points-required' | translate}} + + + {{ 'tenant-profile.max-transport-data-points-range' | translate}} + + + + tenant-profile.max-r-e-executions + + + {{ 'tenant-profile.max-r-e-executions-required' | translate}} + + + {{ 'tenant-profile.max-r-e-executions-range' | translate}} + + + + tenant-profile.max-j-s-executions + + + {{ 'tenant-profile.max-j-s-executions-required' | translate}} + + + {{ 'tenant-profile.max-j-s-executions-range' | translate}} + + + + tenant-profile.max-d-p-storage-days + + + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} + + + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} + + + + tenant-profile.max-rule-node-executions-per-message + + + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} + + + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} + + + + tenant-profile.transport-tenant-msg-rate-limit + + + + tenant-profile.transport-tenant-telemetry-msg-rate-limit + + + + tenant-profile.transport-tenant-telemetry-data-points-rate-limit + + + + tenant-profile.transport-device-msg-rate-limit + + + + tenant-profile.transport-device-telemetry-msg-rate-limit + + + + tenant-profile.transport-device-telemetry-data-points-rate-limit + + +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts new file mode 100644 index 0000000000..f83b59c35a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -0,0 +1,115 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { + DefaultTenantProfileConfiguration, + TenantProfileConfiguration, + TenantProfileType +} from '@shared/models/tenant.model'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-default-tenant-profile-configuration', + templateUrl: './default-tenant-profile-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DefaultTenantProfileConfigurationComponent), + multi: true + }] +}) +export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { + + defaultTenantProfileConfigurationFormGroup: FormGroup; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private fb: FormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.defaultTenantProfileConfigurationFormGroup = this.fb.group({ + configuration: this.fb.group({ + maxDevices: [null, [Validators.required, Validators.min(0)]], + maxAssets: [null, [Validators.required, Validators.min(0)]], + transportTenantMsgRateLimit: [null, []], + transportTenantTelemetryMsgRateLimit: [null, []], + transportTenantTelemetryDataPointsRateLimit: [null, []], + transportDeviceMsgRateLimit: [null, []], + transportDeviceTelemetryMsgRateLimit: [null, []], + transportDeviceTelemetryDataPointsRateLimit: [null, []], + maxTransportMessages: [null, [Validators.required, Validators.min(0)]], + maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], + maxREExecutions: [null, [Validators.required, Validators.min(0)]], + maxJSExecutions: [null, [Validators.required, Validators.min(0)]], + maxDPStorageDays: [null, [Validators.required, Validators.min(0)]], + maxRuleNodeExecutionsPerMessage: [null, [Validators.required, Validators.min(0)]] + }) + }); + this.defaultTenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.defaultTenantProfileConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.defaultTenantProfileConfigurationFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: DefaultTenantProfileConfiguration | null): void { + if (isDefinedAndNotNull(value)) { + this.defaultTenantProfileConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + } + } + + private updateModel() { + let configuration: TenantProfileConfiguration = null; + if (this.defaultTenantProfileConfigurationFormGroup.valid) { + configuration = this.defaultTenantProfileConfigurationFormGroup.getRawValue().configuration; + configuration.type = TenantProfileType.DEFAULT; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html new file mode 100644 index 0000000000..3204b7d2fb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html @@ -0,0 +1,27 @@ + +
+
+ + + + +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts new file mode 100644 index 0000000000..4f5fa989d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts @@ -0,0 +1,103 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { deepClone } from '@core/utils'; +import { TenantProfileConfiguration, TenantProfileType } from '@shared/models/tenant.model'; + +@Component({ + selector: 'tb-tenant-profile-configuration', + templateUrl: './tenant-profile-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => TenantProfileConfigurationComponent), + multi: true + }] +}) +export class TenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { + + tenantProfileType = TenantProfileType; + + tenantProfileConfigurationFormGroup: FormGroup; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + type: TenantProfileType; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private fb: FormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.tenantProfileConfigurationFormGroup = this.fb.group({ + configuration: [null, Validators.required] + }); + this.tenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.tenantProfileConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.tenantProfileConfigurationFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: TenantProfileConfiguration | null): void { + this.type = value?.type; + const configuration = deepClone(value); + if (configuration) { + delete configuration.type; + } + this.tenantProfileConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + } + + private updateModel() { + let configuration: TenantProfileConfiguration = null; + if (this.tenantProfileConfigurationFormGroup.valid) { + configuration = this.tenantProfileConfigurationFormGroup.getRawValue().configuration; + configuration.type = this.type; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/shared/models/alias.models.ts b/ui-ngx/src/app/shared/models/alias.models.ts index 35b52435a2..f437931f27 100644 --- a/ui-ngx/src/app/shared/models/alias.models.ts +++ b/ui-ngx/src/app/shared/models/alias.models.ts @@ -28,6 +28,7 @@ export enum AliasFilterType { assetType = 'assetType', deviceType = 'deviceType', entityViewType = 'entityViewType', + apiUsageState = 'apiUsageState', relationsQuery = 'relationsQuery', assetSearchQuery = 'assetSearchQuery', deviceSearchQuery = 'deviceSearchQuery', @@ -43,6 +44,7 @@ export const aliasFilterTypeTranslationMap = new Map( [ AliasFilterType.assetType, 'alias.filter-type-asset-type' ], [ AliasFilterType.deviceType, 'alias.filter-type-device-type' ], [ AliasFilterType.entityViewType, 'alias.filter-type-entity-view-type' ], + [ AliasFilterType.apiUsageState, 'alias.filter-type-apiUsageState' ], [ AliasFilterType.relationsQuery, 'alias.filter-type-relations-query' ], [ AliasFilterType.assetSearchQuery, 'alias.filter-type-asset-search-query' ], [ AliasFilterType.deviceSearchQuery, 'alias.filter-type-device-search-query' ], @@ -106,6 +108,10 @@ export interface EntitySearchQueryFilter { fetchLastLevelOnly?: boolean; } +export interface ApiUsageStateFilter { + +} + export interface AssetSearchQueryFilter extends EntitySearchQueryFilter { assetTypes?: string[]; } @@ -129,7 +135,8 @@ export type EntityFilters = RelationsQueryFilter & AssetSearchQueryFilter & DeviceSearchQueryFilter & - EntityViewSearchQueryFilter; + EntityViewSearchQueryFilter & + EntitySearchQueryFilter; export interface EntityAliasFilter extends EntityFilters { type?: AliasFilterType; diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 378dca7c25..ad946efdb7 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -19,8 +19,59 @@ import { TenantId } from './id/tenant-id'; import { TenantProfileId } from '@shared/models/id/tenant-profile-id'; import { BaseData } from '@shared/models/base-data'; +export enum TenantProfileType { + DEFAULT = 'DEFAULT' +} + +export interface DefaultTenantProfileConfiguration { + maxDevices: number; + maxAssets: number; + + transportTenantMsgRateLimit?: string; + transportTenantTelemetryMsgRateLimit?: string; + transportTenantTelemetryDataPointsRateLimit?: string; + transportDeviceMsgRateLimit?: string; + transportDeviceTelemetryMsgRateLimit?: string; + transportDeviceTelemetryDataPointsRateLimit?: string; + + maxTransportMessages: number; + maxTransportDataPoints: number; + maxREExecutions: number; + maxJSExecutions: number; + maxDPStorageDays: number; + maxRuleNodeExecutionsPerMessage: number; +} + +export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; + +export interface TenantProfileConfiguration extends TenantProfileConfigurations { + type: TenantProfileType; +} + +export function createTenantProfileConfiguration(type: TenantProfileType): TenantProfileConfiguration { + let configuration: TenantProfileConfiguration = null; + if (type) { + switch (type) { + case TenantProfileType.DEFAULT: + const defaultConfiguration: DefaultTenantProfileConfiguration = { + maxDevices: 0, + maxAssets: 0, + maxTransportMessages: 0, + maxTransportDataPoints: 0, + maxREExecutions: 0, + maxJSExecutions: 0, + maxDPStorageDays: 0, + maxRuleNodeExecutionsPerMessage: 0 + }; + configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; + break; + } + } + return configuration; +} + export interface TenantProfileData { - [key: string]: string; + configuration: TenantProfileConfiguration; } export interface TenantProfile extends BaseData { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index e3b7b4b81d..020d18ef39 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -290,6 +290,7 @@ "filter-type-device-search-query-description": "Devices with types {{deviceTypes}} that have {{relationType}} relation {{direction}} {{rootEntity}}", "filter-type-entity-view-search-query": "Entity view search query", "filter-type-entity-view-search-query-description": "Entity views with types {{entityViewTypes}} that have {{relationType}} relation {{direction}} {{rootEntity}}", + "filter-type-apiUsageState": "Api Usage State", "entity-filter": "Entity filter", "resolve-multiple": "Resolve as multiple entities", "filter-type": "Filter type", @@ -1925,6 +1926,7 @@ "name": "Name", "name-required": "Name is required.", "data": "Profile data", + "profile-configuration": "Profile configuration", "description": "Description", "default": "Default", "delete-tenant-profile-title": "Are you sure you want to delete the tenant profile '{{tenantProfileName}}'?", @@ -1934,7 +1936,37 @@ "set-default-tenant-profile-title": "Are you sure you want to make the tenant profile '{{tenantProfileName}}' default?", "set-default-tenant-profile-text": "After the confirmation the tenant profile will be marked as default and will be used for new tenants with no profile specified.", "no-tenant-profiles-found": "No tenant profiles found.", - "create-new-tenant-profile": "Create a new one!" + "create-new-tenant-profile": "Create a new one!", + "maximum-devices": "Maximum number of devices (0 - unlimited)", + "maximum-devices-required": "Maximum number of devices is required.", + "maximum-devices-range": "Minimum number of devices can't be negative", + "maximum-assets": "Maximum number of assets (0 - unlimited)", + "maximum-assets-required": "Maximum number of assets is required.", + "maximum-assets-range": "Maximum number of assets can't be negative", + "transport-tenant-msg-rate-limit": "Transport tenant messages rate limit.", + "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages rate limit.", + "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points rate limit.", + "transport-device-msg-rate-limit": "Transport device messages rate limit.", + "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", + "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", + "max-transport-messages-required": "Maximum number of transport messages is required.", + "max-transport-messages-range": "Maximum number of transport messages can't be negative", + "max-transport-data-points": "Maximum number of transport data points (0 - unlimited)", + "max-transport-data-points-required": "Maximum number of transport data points is required.", + "max-transport-data-points-range": "Minimum number of transport data points can't be negative", + "max-r-e-executions": "Maximum number of Rule Engine executions (0 - unlimited)", + "max-r-e-executions-required": "Maximum number of Rule Engine executions is required.", + "max-r-e-executions-range": "Minimum number of Rule Engine executions can't be negative", + "max-j-s-executions": "Maximum number of JavaScript executions (0 - unlimited)", + "max-j-s-executions-required": "Maximum number of JavaScript executions is required.", + "max-j-s-executions-range": "Minimum number of JavaScript executions can't be negative", + "max-d-p-storage-days": "Maximum number of data points storage days (0 - unlimited)", + "max-d-p-storage-days-required": "Maximum number of data points storage days is required.", + "max-d-p-storage-days-range": "Minimum number of data points storage days can't be negative", + "max-rule-node-executions-per-message": "Maximum number of rule node executions per message (0 - unlimited)", + "max-rule-node-executions-per-message-required": "Maximum number of rule node executions per message is required.", + "max-rule-node-executions-per-message-range": "Minimum number of rule node executions per message can't be negative" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }",