From 2e694e1d649d46c514d0bba86e3bc27054889d74 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 19 Oct 2020 15:39:15 +0300 Subject: [PATCH 01/30] Dao layer for UsageRecords --- .../dao/usagerecord/UsageRecordService.java | 28 +++++ .../server/common/data/EntityType.java | 2 +- .../server/common/data/UsageRecord.java | 65 ++++++++++ .../server/common/data/UsageRecordKey.java | 27 +++++ .../server/common/data/id/UsageRecordId.java | 42 +++++++ .../server/dao/model/ModelConstants.java | 8 ++ .../dao/model/sql/UsageRecordEntity.java | 93 +++++++++++++++ .../sql/usagerecord/JpaUsageRecordDao.java | 60 ++++++++++ .../usagerecord/UsageRecordRepository.java | 43 +++++++ .../server/dao/tenant/TenantServiceImpl.java | 6 + .../dao/usagerecord/UsageRecordDao.java | 51 ++++++++ .../usagerecord/UsageRecordServiceImpl.java | 112 ++++++++++++++++++ .../resources/sql/schema-entities-hsql.sql | 9 ++ .../main/resources/sql/schema-entities.sql | 9 ++ .../dao/service/AbstractServiceTest.java | 4 + .../service/BaseUsageRecordServiceTest.java | 56 +++++++++ .../sql/UsageRecordServiceSqlTest.java | 24 ++++ .../resources/sql/hsql/drop-all-tables.sql | 1 + .../resources/sql/psql/drop-all-tables.sql | 1 + .../sql/timescale/drop-all-tables.sql | 1 + .../org/thingsboard/mqtt/MqttClientImpl.java | 1 + 21 files changed, 642 insertions(+), 1 deletion(-) create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java new file mode 100644 index 0000000000..55adc257ad --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.id.TenantId; + +public interface UsageRecordService { + + UsageRecord findTenantUsageRecord(TenantId tenantId); + + void deleteUsageRecordsByTenantId(TenantId tenantId); + + void createDefaultUsageRecord(TenantId id); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index cfe12a14cb..11bf45be76 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data; * @author Andrew Shvayka */ public enum EntityType { - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, USAGE_RECORD; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java b/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java new file mode 100644 index 0000000000..e96cf2e452 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UsageRecordId; +import org.thingsboard.server.common.data.id.UserId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class UsageRecord extends BaseData implements HasTenantId { + + private static final long serialVersionUID = 8250339805336035966L; + + private TenantId tenantId; + private EntityId entityId; + + public UsageRecord() { + super(); + } + + public UsageRecord(UsageRecordId id) { + super(id); + } + + public UsageRecord(UsageRecord ur) { + super(ur); + this.tenantId = ur.getTenantId(); + this.entityId = ur.getEntityId(); + } + + @Override + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + public EntityId getEntityId() { + return entityId; + } + + public void setEntityId(EntityId entityId) { + this.entityId = entityId; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java new file mode 100644 index 0000000000..74a54cf713 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public enum UsageRecordKey { + + MSG_COUNT, + MSG_BYTES_COUNT, + DP_TRANSPORT_COUNT, + DP_STORAGE_COUNT, + RE_EXEC_COUNT, + JS_EXEC_COUNT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java new file mode 100644 index 0000000000..76cfaaedf6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public class UsageRecordId extends UUIDBased implements EntityId { + + @JsonCreator + public UsageRecordId(@JsonProperty("id") UUID id) { + super(id); + } + + public static UsageRecordId fromString(String userId) { + return new UsageRecordId(UUID.fromString(userId)); + } + + @JsonIgnore + @Override + public EntityType getEntityType() { + return EntityType.USAGE_RECORD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 14ab9ed9e0..3ee942b0be 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -438,6 +438,14 @@ public class ModelConstants { public static final String OAUTH2_TEMPLATE_LOGIN_BUTTON_LABEL_PROPERTY = OAUTH2_LOGIN_BUTTON_LABEL_PROPERTY; public static final String OAUTH2_TEMPLATE_HELP_LINK_PROPERTY = "help_link"; + /** + * Usage Record constants. + */ + public static final String UR_TABLE_NAME = "usage_record"; + public static final String UR_TENANT_ID_COLUMN = TENANT_ID_PROPERTY; + public static final String UR_ENTITY_TYPE_COLUMN = ENTITY_TYPE_COLUMN; + public static final String UR_ENTITY_ID_COLUMN = ENTITY_ID_COLUMN; + /** * Cassandra attributes and timeseries constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java new file mode 100644 index 0000000000..be5cc40621 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UsageRecordId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.SearchTextEntity; +import org.thingsboard.server.dao.util.mapping.JsonStringType; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; +import java.util.UUID; + +/** + * Created by Valerii Sosliuk on 4/21/2017. + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@TypeDef(name = "json", typeClass = JsonStringType.class) +@Table(name = ModelConstants.UR_TABLE_NAME) +public class UsageRecordEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = ModelConstants.UR_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = ModelConstants.UR_ENTITY_TYPE_COLUMN) + private String entityType; + + @Column(name = ModelConstants.UR_ENTITY_ID_COLUMN) + private UUID entityId; + + public UsageRecordEntity() { + } + + public UsageRecordEntity(UsageRecord ur) { + if (ur.getId() != null) { + this.setUuid(ur.getId().getId()); + } + this.setCreatedTime(ur.getCreatedTime()); + if (ur.getTenantId() != null) { + this.tenantId = ur.getTenantId().getId(); + } + if (ur.getEntityId() != null) { + this.entityType = ur.getEntityId().getEntityType().name(); + this.entityId = ur.getEntityId().getId(); + } + } + + @Override + public UsageRecord toData() { + UsageRecord ur = new UsageRecord(new UsageRecordId(this.getUuid())); + ur.setCreatedTime(createdTime); + if (tenantId != null) { + ur.setTenantId(new TenantId(tenantId)); + } + if (entityId != null) { + ur.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + return ur; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java new file mode 100644 index 0000000000..8d54e6b53b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.usagerecord; + +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sql.UsageRecordEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.usagerecord.UsageRecordDao; + +import java.util.UUID; + +/** + * @author Andrii Shvaika + */ +@Component +public class JpaUsageRecordDao extends JpaAbstractDao implements UsageRecordDao { + + private final UsageRecordRepository usageRecordRepository; + + public JpaUsageRecordDao(UsageRecordRepository usageRecordRepository) { + this.usageRecordRepository = usageRecordRepository; + } + + @Override + protected Class getEntityClass() { + return UsageRecordEntity.class; + } + + @Override + protected CrudRepository getCrudRepository() { + return usageRecordRepository; + } + + @Override + public UsageRecord findTenantUsageRecord(UUID tenantId) { + return DaoUtil.getData(usageRecordRepository.findByTenantId(tenantId)); + } + + @Override + public void deleteUsageRecordsByTenantId(TenantId tenantId) { + usageRecordRepository.deleteUsageRecordsByTenantId(tenantId.getId()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java new file mode 100644 index 0000000000..dc2b0d9ad7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.usagerecord; + +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.sql.UsageRecordEntity; +import org.thingsboard.server.dao.model.sql.UserEntity; + +import java.util.UUID; + +/** + * @author Valerii Sosliuk + */ +public interface UsageRecordRepository extends CrudRepository { + + @Query("SELECT ur FROM UsageRecordEntity ur WHERE ur.tenantId = :tenantId " + + "AND ur.entityId = :tenantId AND ur.entityType = 'TENANT' ") + UsageRecordEntity findByTenantId(@Param("tenantId") UUID tenantId); + + @Transactional + @Modifying + @Query("DELETE FROM UsageRecordEntity ur WHERE ur.tenantId = :tenantId") + void deleteUsageRecordsByTenantId(@Param("tenantId") UUID tenantId); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 52761184fd..bd9ee321e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.usagerecord.UsageRecordService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -72,6 +73,9 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private DeviceProfileService deviceProfileService; + @Autowired + private UsageRecordService usageRecordService; + @Autowired private EntityViewService entityViewService; @@ -117,6 +121,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe Tenant savedTenant = tenantDao.save(tenant.getId(), tenant); if (tenant.getId() == null) { deviceProfileService.createDefaultDeviceProfile(savedTenant.getId()); + usageRecordService.createDefaultUsageRecord(savedTenant.getId()); } return savedTenant; } @@ -134,6 +139,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); userService.deleteTenantAdmins(tenantId); ruleChainService.deleteRuleChainsByTenantId(tenantId); + usageRecordService.deleteUsageRecordsByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); deleteEntityRelations(tenantId, tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java new file mode 100644 index 0000000000..3b71e4ce20 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +import java.util.UUID; + +public interface UsageRecordDao extends Dao { + + /** + * Save or update usage record object + * + * @param usageRecord the usage record + * @return saved usage record entity + */ + UsageRecord save(TenantId tenantId, UsageRecord usageRecord); + + /** + * Find usage record by tenantId. + * + * @param tenantId the tenantId + * @return the corresponding usage record + */ + UsageRecord findTenantUsageRecord(UUID tenantId); + + /** + * Delete usage record by tenantId. + * + * @param tenantId the tenantId + */ + void deleteUsageRecordsByTenantId(TenantId tenantId); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java new file mode 100644 index 0000000000..fdaa38a1a2 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserCredentialsId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.tenant.TenantDao; + +import java.util.HashMap; +import java.util.Map; + +import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; +import static org.thingsboard.server.dao.service.Validator.validateString; + +@Service +@Slf4j +public class UsageRecordServiceImpl extends AbstractEntityService implements UsageRecordService { + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + + private final UsageRecordDao usageRecordDao; + private final TenantDao tenantDao; + + public UsageRecordServiceImpl(TenantDao tenantDao, UsageRecordDao usageRecordDao) { + this.tenantDao = tenantDao; + this.usageRecordDao = usageRecordDao; + } + + @Override + public void deleteUsageRecordsByTenantId(TenantId tenantId) { + log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + usageRecordDao.deleteUsageRecordsByTenantId(tenantId); + } + + @Override + public void createDefaultUsageRecord(TenantId tenantId) { + log.trace("Executing createDefaultUsageRecord [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + UsageRecord usageRecord = new UsageRecord(); + usageRecord.setTenantId(tenantId); + usageRecord.setEntityId(tenantId); + usageRecordValidator.validate(usageRecord, UsageRecord::getTenantId); + usageRecordDao.save(usageRecord.getTenantId(), usageRecord); + } + + @Override + public UsageRecord findTenantUsageRecord(TenantId tenantId) { + log.trace("Executing findTenantUsageRecord, tenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + return usageRecordDao.findTenantUsageRecord(tenantId.getId()); + } + + private DataValidator usageRecordValidator = + new DataValidator() { + @Override + protected void validateDataImpl(TenantId requestTenantId, UsageRecord usageRecord) { + if (usageRecord.getTenantId() == null) { + throw new DataValidationException("UsageRecord should be assigned to tenant!"); + } else { + Tenant tenant = tenantDao.findById(requestTenantId, usageRecord.getTenantId().getId()); + if (tenant == null) { + throw new DataValidationException("Asset is referencing to non-existent tenant!"); + } + } + if (usageRecord.getEntityId() == null) { + throw new DataValidationException("UsageRecord should be assigned to entity!"); + } else if (!EntityType.TENANT.equals(usageRecord.getEntityId().getEntityType())) { + throw new DataValidationException("Only Tenant Usage Records are supported!"); + } else if (!usageRecord.getTenantId().getId().equals(usageRecord.getEntityId().getId())) { + throw new DataValidationException("Can't assign one Usage Record to multiple tenants!"); + } + } + }; + +} diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 41d9e3ac81..e3cbbda351 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -404,3 +404,12 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( help_link varchar(255), CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); + +CREATE TABLE IF NOT EXISTS usage_record ( + id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + entity_type varchar(32), + entity_id uuid, + CONSTRAINT usage_record_unq_key UNIQUE (tenant_id, entity_id) +); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index e1ffba98eb..6cd3adb337 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -431,6 +431,15 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); +CREATE TABLE IF NOT EXISTS usage_record ( + id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + entity_type varchar(32), + entity_id uuid, + CONSTRAINT usage_record_unq_key UNIQUE (tenant_id, entity_id) +); + CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index cf6f574180..8da1727631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -61,6 +61,7 @@ import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.UsageRecordService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -85,6 +86,9 @@ public abstract class AbstractServiceTest { @Autowired protected UserService userService; + @Autowired + protected UsageRecordService usageRecordService; + @Autowired protected AdminSettingsService adminSettingsService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java new file mode 100644 index 0000000000..28e0a6d54a --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + + +public abstract class BaseUsageRecordServiceTest extends AbstractServiceTest { + + private TenantId tenantId; + + @Before + public void before() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + Assert.assertNotNull(savedTenant); + tenantId = savedTenant.getId(); + } + + @After + public void after() { + tenantService.deleteTenant(tenantId); + } + + @Test + public void testFindUsageRecordByTenantId() { + UsageRecord usageRecord = usageRecordService.findTenantUsageRecord(tenantId); + Assert.assertNotNull(usageRecord); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java new file mode 100644 index 0000000000..8275b92044 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.sql; + +import org.thingsboard.server.dao.service.BaseUsageRecordServiceTest; +import org.thingsboard.server.dao.service.BaseUserServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@DaoSqlTest +public class UsageRecordServiceSqlTest extends BaseUsageRecordServiceTest { +} diff --git a/dao/src/test/resources/sql/hsql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql index 160049a144..13c1eba943 100644 --- a/dao/src/test/resources/sql/hsql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/hsql/drop-all-tables.sql @@ -27,4 +27,5 @@ DROP TABLE IF EXISTS rule_chain; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS usage_record; DROP FUNCTION IF EXISTS to_uuid; diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index e277c39403..b8ab69c758 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -28,3 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS usage_record; \ No newline at end of file diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index da7eaff876..4333bc33ee 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -28,3 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; +DROP TABLE IF EXISTS usage_record; \ No newline at end of file diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index aef2cad684..08e6fc4f71 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -340,6 +340,7 @@ final class MqttClientImpl implements MqttClient { MqttPublishVariableHeader variableHeader = new MqttPublishVariableHeader(topic, getNewMessageId().messageId()); MqttPublishMessage message = new MqttPublishMessage(fixedHeader, variableHeader, payload); MqttPendingPublish pendingPublish = new MqttPendingPublish(variableHeader.packetId(), future, payload.retain(), message, qos); + ChannelFuture channelFuture = this.sendAndFlushPacket(message); if (channelFuture != null) { From 24f5872adf3f50f36629ef0d5fb3b791c92be838 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 19 Oct 2020 18:00:20 +0300 Subject: [PATCH 02/30] Queue interfaces with no implementation --- .../queue/provider/TbCoreQueueFactory.java | 13 ++- .../provider/TbQueueProducerProvider.java | 11 ++ .../provider/TbRuleEngineQueueFactory.java | 2 +- .../provider/TbTransportQueueFactory.java | 10 +- .../TbTransportQueueProducerProvider.java | 7 ++ .../TbUsageStatsClientQueueFactory.java | 29 +++++ .../scheduler/DefaultSchedulerComponent.java | 61 ++++++++++ .../queue/scheduler/SchedulerComponent.java | 32 ++++++ .../usagestats/DefaultTbUsageStatsClient.java | 106 ++++++++++++++++++ .../queue/usagestats/TbUsageStatsClient.java | 26 +++++ common/queue/src/main/proto/queue.proto | 13 +++ .../transport/mqtt/MqttTransportHandler.java | 5 +- .../common/transport/TransportContext.java | 3 + .../common/transport/TransportService.java | 8 +- .../service/DefaultTransportService.java | 24 ++-- 15 files changed, 318 insertions(+), 32 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 59a8f449b0..d009757997 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -33,7 +34,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -77,6 +78,16 @@ public interface TbCoreQueueFactory { */ TbQueueConsumer> createToCoreMsgConsumer(); + /** + * Used to consume messages about usage statistics by TB Core Service + * + * @return + */ + default TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + //TODO: implement + return null; + } + /** * Used to consume high priority messages by TB Core Service * diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 34c6ee577f..fa20f11960 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -63,4 +64,14 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + default TbQueueProducer> getTbUsageStatsMsgProducer() { + //TODO: implement + return null; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 561b3e84ea..2f80bbce45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -32,7 +32,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index dc1d2c449c..bd6c6e0584 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -15,17 +15,17 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbTransportQueueFactory { +public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index bced3f83ac..9d11aa727b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -30,6 +30,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider private final TbTransportQueueFactory tbQueueProvider; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; + private TbQueueProducer> toUsageStats; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -39,6 +40,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public void init() { this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override @@ -65,4 +67,9 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbCoreNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java new file mode 100644 index 0000000000..1811877338 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TbUsageStatsClientQueueFactory { + + default TbQueueProducer> createToUsageStatsServiceMsgProducer() { + //TODO: implement + return null; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java new file mode 100644 index 0000000000..02ce60496e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +@Component +public class DefaultSchedulerComponent implements SchedulerComponent{ + + protected ScheduledExecutorService schedulerExecutor; + + @PostConstruct + public void init(){ + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("queue-scheduler")); + } + + @PreDestroy + public void destroy() { + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + } + + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(command, delay, unit); + } + + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + return schedulerExecutor.schedule(callable, delay, unit); + } + + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { + return schedulerExecutor.scheduleAtFixedRate(command, initialDelay, period, unit); + } + + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { + return schedulerExecutor.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java new file mode 100644 index 0000000000..3c12be43a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/scheduler/SchedulerComponent.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.scheduler; + +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public interface SchedulerComponent { + + ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit); + + ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit); + + ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit); + + ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java new file mode 100644 index 0000000000..87eb5e767a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.UsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; + +import javax.annotation.PostConstruct; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@Component +public class DefaultTbUsageStatsClient implements TbUsageStatsClient { + + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + @Value("${usage.stats.report.interval:600}") + private int interval; + + private final ConcurrentMap[] values = new ConcurrentMap[UsageRecordKey.values().length]; + private final PartitionService partitionService; + private final SchedulerComponent scheduler; + private final TbQueueProducerProvider producerProvider; + private TbQueueProducer> msgProducer; + + public DefaultTbUsageStatsClient(PartitionService partitionService, SchedulerComponent scheduler, TbQueueProducerProvider producerProvider) { + this.partitionService = partitionService; + this.scheduler = scheduler; + this.producerProvider = producerProvider; + } + + @PostConstruct + private void init() { + if (enabled) { + msgProducer = this.producerProvider.getTbUsageStatsMsgProducer(); + for (UsageRecordKey key : UsageRecordKey.values()) { + values[key.ordinal()] = new ConcurrentHashMap<>(); + } + scheduler.scheduleWithFixedDelay(this::reportStats, new Random().nextInt(interval), interval, TimeUnit.SECONDS); + } + } + + private void reportStats() { + ConcurrentMap report = new ConcurrentHashMap<>(); + + for (UsageRecordKey key : UsageRecordKey.values()) { + values[key.ordinal()].forEach(((tenantId, atomicLong) -> { + long value = atomicLong.getAndSet(0); + if (value > 0) { + ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> { + ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder(); + msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + return msg; + }); + msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build()); + } + })); + } + + report.forEach(((tenantId, builder) -> { + //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId); + msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); + })); + } + + @Override + public void report(TenantId tenantId, UsageRecordKey key, long value) { + if (enabled) { + ConcurrentMap map = values[key.ordinal()]; + AtomicLong atomicValue = map.computeIfAbsent(tenantId, id -> new AtomicLong()); + atomicValue.addAndGet(value); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java new file mode 100644 index 0000000000..e4e659b5c5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.usagestats; + +import org.thingsboard.server.common.data.UsageRecordKey; +import org.thingsboard.server.common.data.id.TenantId; + +public interface TbUsageStatsClient { + + void report(TenantId tenantId, UsageRecordKey key, long value); + + +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 01d8ca4524..f716bd90ae 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -532,3 +532,16 @@ message ToTransportMsg { EntityDeleteMsg entityDeleteMsg = 9; ProvisionDeviceResponseMsg provisionResponse = 10; } + +message UsageStatsKVProto{ + string key = 1; + int64 value = 2; +} + +message ToUsageStatsServiceMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; + repeated UsageStatsKVProto values = 5; +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 3723b19f6d..2c4b2db7df 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -57,6 +57,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.session.DeviceSessionCtx; import org.thingsboard.server.transport.mqtt.session.GatewaySessionHandler; @@ -97,6 +98,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private final UUID sessionId; private final MqttTransportContext context; private final TransportService transportService; + private final SchedulerComponent scheduler; private final SslHandler sslHandler; private final ConcurrentMap mqttQoSMap; @@ -108,6 +110,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.sessionId = UUID.randomUUID(); this.context = context; this.transportService = context.getTransportService(); + this.scheduler = context.getScheduler(); this.sslHandler = sslHandler; this.mqttQoSMap = new ConcurrentHashMap<>(); this.deviceSessionCtx = new DeviceSessionCtx(sessionId, mqttQoSMap, context); @@ -333,7 +336,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } else { deviceSessionCtx.getContext().getProtoMqttAdaptor().convertToPublish(deviceSessionCtx, provisionResponseMsg).ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); } - transportService.getSchedulerExecutor().schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); + scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); } catch (Exception e) { log.trace("[{}] Failed to convert device attributes response to MQTT msg", sessionId, e); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index 66809f9e61..4b7677dd69 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -44,6 +45,8 @@ public abstract class TransportContext { private TransportService transportService; @Autowired private TbServiceInfoProvider serviceInfoProvider; + @Autowired + private SchedulerComponent scheduler; @Getter private ExecutorService executor; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 8f57db3897..4872d8e477 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -17,16 +17,14 @@ package org.thingsboard.server.common.transport; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitType; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; @@ -42,8 +40,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCre import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import java.util.concurrent.ScheduledExecutorService; - /** * Created by ashvayka on 04.10.18. */ @@ -92,8 +88,6 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); - ScheduledExecutorService getSchedulerExecutor(); - void registerAsyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener); void registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3f37685383..774a437e39 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -23,7 +23,6 @@ import com.google.gson.JsonObject; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DeviceProfile; @@ -53,7 +52,6 @@ import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.limits.TransportRateLimit; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.limits.TransportRateLimitType; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; @@ -79,6 +77,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbTransportComponent; import javax.annotation.PostConstruct; @@ -94,7 +93,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -126,6 +124,7 @@ public class DefaultTransportService implements TransportService { private final TransportTenantProfileCache tenantProfileCache; private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; + private final SchedulerComponent scheduler; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -136,7 +135,6 @@ public class DefaultTransportService implements TransportService { protected MessagesStats tbCoreProducerStats; protected MessagesStats transportApiStats; - protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; private ExecutorService mainConsumerExecutor; @@ -152,7 +150,7 @@ public class DefaultTransportService implements TransportService { StatsFactory statsFactory, TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, - TransportRateLimitService rateLimitService, DataDecodingEncodingService dataDecodingEncodingService) { + TransportRateLimitService rateLimitService, DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; @@ -162,6 +160,7 @@ public class DefaultTransportService implements TransportService { this.tenantProfileCache = tenantProfileCache; this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; + this.scheduler = scheduler; } @PostConstruct @@ -169,9 +168,8 @@ public class DefaultTransportService implements TransportService { this.ruleEngineProducerStats = statsFactory.createMessagesStats(StatsType.RULE_ENGINE.getName() + ".producer"); this.tbCoreProducerStats = statsFactory.createMessagesStats(StatsType.CORE.getName() + ".producer"); this.transportApiStats = statsFactory.createMessagesStats(StatsType.TRANSPORT.getName() + ".producer"); - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + this.scheduler.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); transportApiRequestTemplate.setMessagesStats(transportApiStats); ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); @@ -217,9 +215,6 @@ public class DefaultTransportService implements TransportService { if (transportNotificationsConsumer != null) { transportNotificationsConsumer.unsubscribe(); } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); - } if (transportCallbackExecutor != null) { transportCallbackExecutor.shutdownNow(); } @@ -231,11 +226,6 @@ public class DefaultTransportService implements TransportService { } } - @Override - public ScheduledExecutorService getSchedulerExecutor() { - return this.schedulerExecutor; - } - @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); @@ -488,7 +478,7 @@ public class DefaultTransportService implements TransportService { sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); String requestId = sessionId + "-" + msg.getRequestId(); toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); - schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); + scheduler.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); } } @@ -561,7 +551,7 @@ public class DefaultTransportService implements TransportService { SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); - ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { + ScheduledFuture executorFuture = scheduler.schedule(() -> { listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); deregisterSession(sessionInfo); }, timeout, TimeUnit.MILLISECONDS); From 5eb91c17f6790ed973d8c44a23d7d5a88284a8c8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 19 Oct 2020 18:25:34 +0300 Subject: [PATCH 03/30] Usage records --- .../queue/DefaultTbCoreConsumerService.java | 79 +++++++++++++++++-- .../DefaultTbUsageStatsService.java | 15 ++++ .../usagestats/TbUsageStatsService.java | 10 +++ 3 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 5d240790f2..3f03b65416 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,20 +17,21 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -61,6 +62,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; +import org.thingsboard.server.service.usagestats.TbUsageStatsService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -70,6 +72,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -88,32 +92,50 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> mainConsumer; private final DeviceStateService stateService; + private final TbUsageStatsService statsService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats; + protected final TbQueueConsumer> usageStatsConsumer; + + protected volatile ExecutorService usageStatsExecutor; + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, - TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { + TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache, + TbUsageStatsService statsService) { super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); + this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer(); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; this.stats = new TbCoreConsumerStats(statsFactory); + this.statsService = statsService; } @PostConstruct public void init() { super.init("tb-core-consumer", "tb-core-notifications-consumer"); + this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer")); } @PreDestroy public void destroy() { super.destroy(); + if (usageStatsExecutor != null) { + usageStatsExecutor.shutdownNow(); + } + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + launchUsageStatsConsumer(); } @Override @@ -223,6 +245,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + while (!stopped) { + try { + List> msgs = usageStatsConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + pendingMap.forEach((id, msg) -> { + log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + handleUsageStats(msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process usge stats: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue())); + } + usageStatsConsumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain usage stats from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new usage stats", e2); + } + } + } + } + log.info("TB Usage Stats Consumer stopped."); + }); + } + + private void handleUsageStats(TbProtoQueueMsg msg, TbCallback callback) { + statsService.process(msg, callback); + } + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java new file mode 100644 index 0000000000..6611507531 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java @@ -0,0 +1,15 @@ +package org.thingsboard.server.service.usagestats; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +@Service +public class DefaultTbUsageStatsService implements TbUsageStatsService { + @Override + public void process(TbProtoQueueMsg msg, TbCallback callback) { + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java new file mode 100644 index 0000000000..35a7051896 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java @@ -0,0 +1,10 @@ +package org.thingsboard.server.service.usagestats; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TbUsageStatsService { + + void process(TbProtoQueueMsg msg, TbCallback callback); +} From b598a7395492151598f8fe956d0022d5b4de33c3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 19 Oct 2020 19:51:58 +0300 Subject: [PATCH 04/30] Renaming of UsageRecord to ApiUsageState --- .../apiusage/DefaultTbUsageStatsService.java | 30 +++++ .../service/apiusage/TbUsageStatsService.java | 25 ++++ .../queue/DefaultTbCoreConsumerService.java | 4 +- .../DefaultTbUsageStatsService.java | 15 --- .../usagestats/TbUsageStatsService.java | 10 -- ...Service.java => ApiUsageStateService.java} | 10 +- ...eRecordKey.java => ApiUsageRecordKey.java} | 2 +- .../{UsageRecord.java => ApiUsageState.java} | 11 +- .../server/common/data/EntityType.java | 2 +- ...sageRecordId.java => ApiUsageStateId.java} | 10 +- .../usagestats/DefaultTbUsageStatsClient.java | 11 +- .../queue/usagestats/TbUsageStatsClient.java | 4 +- .../server/dao/model/ModelConstants.java | 8 +- ...rdEntity.java => ApiUsageStateEntity.java} | 31 ++--- ...tory.java => ApiUsageStateRepository.java} | 15 +-- ...ecordDao.java => JpaApiUsageStateDao.java} | 30 ++--- .../server/dao/tenant/TenantServiceImpl.java | 8 +- .../ApiApiUsageStateServiceImpl.java | 91 ++++++++++++++ ...geRecordDao.java => ApiUsageStateDao.java} | 15 +-- .../usagerecord/UsageRecordServiceImpl.java | 112 ------------------ .../resources/sql/schema-entities-hsql.sql | 4 +- .../main/resources/sql/schema-entities.sql | 4 +- .../dao/service/AbstractServiceTest.java | 8 +- ...java => BaseApiUsageStateServiceTest.java} | 15 +-- ....java => ApiUsageStateServiceSqlTest.java} | 5 +- .../resources/sql/hsql/drop-all-tables.sql | 2 +- .../resources/sql/psql/drop-all-tables.sql | 2 +- .../sql/timescale/drop-all-tables.sql | 2 +- 28 files changed, 234 insertions(+), 252 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java rename common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/{UsageRecordService.java => ApiUsageStateService.java} (73%) rename common/data/src/main/java/org/thingsboard/server/common/data/{UsageRecordKey.java => ApiUsageRecordKey.java} (96%) rename common/data/src/main/java/org/thingsboard/server/common/data/{UsageRecord.java => ApiUsageState.java} (83%) rename common/data/src/main/java/org/thingsboard/server/common/data/id/{UsageRecordId.java => ApiUsageStateId.java} (77%) rename dao/src/main/java/org/thingsboard/server/dao/model/sql/{UsageRecordEntity.java => ApiUsageStateEntity.java} (67%) rename dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/{UsageRecordRepository.java => ApiUsageStateRepository.java} (64%) rename dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/{JpaUsageRecordDao.java => JpaApiUsageStateDao.java} (51%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java rename dao/src/main/java/org/thingsboard/server/dao/usagerecord/{UsageRecordDao.java => ApiUsageStateDao.java} (69%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java rename dao/src/test/java/org/thingsboard/server/dao/service/{BaseUsageRecordServiceTest.java => BaseApiUsageStateServiceTest.java} (68%) rename dao/src/test/java/org/thingsboard/server/dao/service/sql/{UsageRecordServiceSqlTest.java => ApiUsageStateServiceSqlTest.java} (77%) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java new file mode 100644 index 0000000000..e2679d6433 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +@Service +public class DefaultTbUsageStatsService implements TbUsageStatsService { + @Override + public void process(TbProtoQueueMsg msg, TbCallback callback) { + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java new file mode 100644 index 0000000000..3c54479eaf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface TbUsageStatsService { + + void process(TbProtoQueueMsg msg, TbCallback callback); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3f03b65416..a389eefd1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -62,7 +62,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.service.usagestats.TbUsageStatsService; +import org.thingsboard.server.service.apiusage.TbUsageStatsService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java deleted file mode 100644 index 6611507531..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.thingsboard.server.service.usagestats; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -@Service -public class DefaultTbUsageStatsService implements TbUsageStatsService { - @Override - public void process(TbProtoQueueMsg msg, TbCallback callback) { - - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java deleted file mode 100644 index 35a7051896..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.thingsboard.server.service.usagestats; - -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -public interface TbUsageStatsService { - - void process(TbProtoQueueMsg msg, TbCallback callback); -} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java similarity index 73% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index 55adc257ad..ad6c73f63e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -15,14 +15,14 @@ */ package org.thingsboard.server.dao.usagerecord; -import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.TenantId; -public interface UsageRecordService { +public interface ApiUsageStateService { - UsageRecord findTenantUsageRecord(TenantId tenantId); + ApiUsageState findTenantApiUsageState(TenantId tenantId); - void deleteUsageRecordsByTenantId(TenantId tenantId); + void deleteApiUsageStateByTenantId(TenantId tenantId); - void createDefaultUsageRecord(TenantId id); + void createDefaultApiUsageState(TenantId id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java similarity index 96% rename from common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java rename to common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java index 74a54cf713..9c8690a88f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecordKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.common.data; -public enum UsageRecordKey { +public enum ApiUsageRecordKey { MSG_COUNT, MSG_BYTES_COUNT, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java similarity index 83% rename from common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java rename to common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java index e96cf2e452..796e434c8a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/UsageRecord.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -19,27 +19,26 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UsageRecordId; -import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; @ToString @EqualsAndHashCode(callSuper = true) -public class UsageRecord extends BaseData implements HasTenantId { +public class ApiUsageState extends BaseData implements HasTenantId { private static final long serialVersionUID = 8250339805336035966L; private TenantId tenantId; private EntityId entityId; - public UsageRecord() { + public ApiUsageState() { super(); } - public UsageRecord(UsageRecordId id) { + public ApiUsageState(ApiUsageStateId id) { super(id); } - public UsageRecord(UsageRecord ur) { + public ApiUsageState(ApiUsageState ur) { super(ur); this.tenantId = ur.getTenantId(); this.entityId = ur.getEntityId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 11bf45be76..9f7fb711e8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data; * @author Andrew Shvayka */ public enum EntityType { - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, USAGE_RECORD; + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java similarity index 77% rename from common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java index 76cfaaedf6..87983f519a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/UsageRecordId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java @@ -22,21 +22,21 @@ import org.thingsboard.server.common.data.EntityType; import java.util.UUID; -public class UsageRecordId extends UUIDBased implements EntityId { +public class ApiUsageStateId extends UUIDBased implements EntityId { @JsonCreator - public UsageRecordId(@JsonProperty("id") UUID id) { + public ApiUsageStateId(@JsonProperty("id") UUID id) { super(id); } - public static UsageRecordId fromString(String userId) { - return new UsageRecordId(UUID.fromString(userId)); + public static ApiUsageStateId fromString(String userId) { + return new ApiUsageStateId(UUID.fromString(userId)); } @JsonIgnore @Override public EntityType getEntityType() { - return EntityType.USAGE_RECORD; + return EntityType.API_USAGE_STATE; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java index 87eb5e767a..534937031c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java @@ -17,13 +17,12 @@ package org.thingsboard.server.queue.usagestats; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.UsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; -import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -46,7 +45,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { @Value("${usage.stats.report.interval:600}") private int interval; - private final ConcurrentMap[] values = new ConcurrentMap[UsageRecordKey.values().length]; + private final ConcurrentMap[] values = new ConcurrentMap[ApiUsageRecordKey.values().length]; private final PartitionService partitionService; private final SchedulerComponent scheduler; private final TbQueueProducerProvider producerProvider; @@ -62,7 +61,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { private void init() { if (enabled) { msgProducer = this.producerProvider.getTbUsageStatsMsgProducer(); - for (UsageRecordKey key : UsageRecordKey.values()) { + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { values[key.ordinal()] = new ConcurrentHashMap<>(); } scheduler.scheduleWithFixedDelay(this::reportStats, new Random().nextInt(interval), interval, TimeUnit.SECONDS); @@ -72,7 +71,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { private void reportStats() { ConcurrentMap report = new ConcurrentHashMap<>(); - for (UsageRecordKey key : UsageRecordKey.values()) { + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { values[key.ordinal()].forEach(((tenantId, atomicLong) -> { long value = atomicLong.getAndSet(0); if (value > 0) { @@ -95,7 +94,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { } @Override - public void report(TenantId tenantId, UsageRecordKey key, long value) { + public void report(TenantId tenantId, ApiUsageRecordKey key, long value) { if (enabled) { ConcurrentMap map = values[key.ordinal()]; AtomicLong atomicValue = map.computeIfAbsent(tenantId, id -> new AtomicLong()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java index e4e659b5c5..4ed850a018 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.queue.usagestats; -import org.thingsboard.server.common.data.UsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.TenantId; public interface TbUsageStatsClient { - void report(TenantId tenantId, UsageRecordKey key, long value); + void report(TenantId tenantId, ApiUsageRecordKey key, long value); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 3ee942b0be..a0f5b45f02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -441,10 +441,10 @@ public class ModelConstants { /** * Usage Record constants. */ - public static final String UR_TABLE_NAME = "usage_record"; - public static final String UR_TENANT_ID_COLUMN = TENANT_ID_PROPERTY; - public static final String UR_ENTITY_TYPE_COLUMN = ENTITY_TYPE_COLUMN; - public static final String UR_ENTITY_ID_COLUMN = ENTITY_ID_COLUMN; + public static final String API_USAGE_STATE_TABLE_NAME = "api_usage_state"; + public static final String API_USAGE_STATE_TENANT_ID_COLUMN = TENANT_ID_PROPERTY; + public static final String API_USAGE_STATE_ENTITY_TYPE_COLUMN = ENTITY_TYPE_COLUMN; + public static final String API_USAGE_STATE_ENTITY_ID_COLUMN = ENTITY_ID_COLUMN; /** * Cassandra attributes and timeseries constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java similarity index 67% rename from dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java index be5cc40621..a06ec142d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UsageRecordEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -15,29 +15,20 @@ */ package org.thingsboard.server.dao.model.sql; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import lombok.EqualsAndHashCode; -import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; -import org.thingsboard.server.common.data.UsageRecord; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UsageRecordId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; import javax.persistence.Table; import java.util.UUID; @@ -48,22 +39,22 @@ import java.util.UUID; @EqualsAndHashCode(callSuper = true) @Entity @TypeDef(name = "json", typeClass = JsonStringType.class) -@Table(name = ModelConstants.UR_TABLE_NAME) -public class UsageRecordEntity extends BaseSqlEntity implements BaseEntity { +@Table(name = ModelConstants.API_USAGE_STATE_TABLE_NAME) +public class ApiUsageStateEntity extends BaseSqlEntity implements BaseEntity { - @Column(name = ModelConstants.UR_TENANT_ID_COLUMN) + @Column(name = ModelConstants.API_USAGE_STATE_TENANT_ID_COLUMN) private UUID tenantId; - @Column(name = ModelConstants.UR_ENTITY_TYPE_COLUMN) + @Column(name = ModelConstants.API_USAGE_STATE_ENTITY_TYPE_COLUMN) private String entityType; - @Column(name = ModelConstants.UR_ENTITY_ID_COLUMN) + @Column(name = ModelConstants.API_USAGE_STATE_ENTITY_ID_COLUMN) private UUID entityId; - public UsageRecordEntity() { + public ApiUsageStateEntity() { } - public UsageRecordEntity(UsageRecord ur) { + public ApiUsageStateEntity(ApiUsageState ur) { if (ur.getId() != null) { this.setUuid(ur.getId().getId()); } @@ -78,8 +69,8 @@ public class UsageRecordEntity extends BaseSqlEntity implements Bas } @Override - public UsageRecord toData() { - UsageRecord ur = new UsageRecord(new UsageRecordId(this.getUuid())); + public ApiUsageState toData() { + ApiUsageState ur = new ApiUsageState(new ApiUsageStateId(this.getUuid())); ur.setCreatedTime(createdTime); if (tenantId != null) { ur.setTenantId(new TenantId(tenantId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java similarity index 64% rename from dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index dc2b0d9ad7..122bb9d966 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/UsageRecordRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -20,24 +20,21 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.common.data.UsageRecord; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.model.sql.UsageRecordEntity; -import org.thingsboard.server.dao.model.sql.UserEntity; +import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import java.util.UUID; /** * @author Valerii Sosliuk */ -public interface UsageRecordRepository extends CrudRepository { +public interface ApiUsageStateRepository extends CrudRepository { - @Query("SELECT ur FROM UsageRecordEntity ur WHERE ur.tenantId = :tenantId " + + @Query("SELECT ur FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId " + "AND ur.entityId = :tenantId AND ur.entityType = 'TENANT' ") - UsageRecordEntity findByTenantId(@Param("tenantId") UUID tenantId); + ApiUsageStateEntity findByTenantId(@Param("tenantId") UUID tenantId); @Transactional @Modifying - @Query("DELETE FROM UsageRecordEntity ur WHERE ur.tenantId = :tenantId") - void deleteUsageRecordsByTenantId(@Param("tenantId") UUID tenantId); + @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") + void deleteApiUsageStateByTenantId(@Param("tenantId") UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java similarity index 51% rename from dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 8d54e6b53b..2c6b2e924d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaUsageRecordDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -17,12 +17,12 @@ package org.thingsboard.server.dao.sql.usagerecord; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.UsageRecord; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.sql.UsageRecordEntity; +import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.usagerecord.UsageRecordDao; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import java.util.UUID; @@ -30,31 +30,31 @@ import java.util.UUID; * @author Andrii Shvaika */ @Component -public class JpaUsageRecordDao extends JpaAbstractDao implements UsageRecordDao { +public class JpaApiUsageStateDao extends JpaAbstractDao implements ApiUsageStateDao { - private final UsageRecordRepository usageRecordRepository; + private final ApiUsageStateRepository apiUsageStateRepository; - public JpaUsageRecordDao(UsageRecordRepository usageRecordRepository) { - this.usageRecordRepository = usageRecordRepository; + public JpaApiUsageStateDao(ApiUsageStateRepository apiUsageStateRepository) { + this.apiUsageStateRepository = apiUsageStateRepository; } @Override - protected Class getEntityClass() { - return UsageRecordEntity.class; + protected Class getEntityClass() { + return ApiUsageStateEntity.class; } @Override - protected CrudRepository getCrudRepository() { - return usageRecordRepository; + protected CrudRepository getCrudRepository() { + return apiUsageStateRepository; } @Override - public UsageRecord findTenantUsageRecord(UUID tenantId) { - return DaoUtil.getData(usageRecordRepository.findByTenantId(tenantId)); + public ApiUsageState findTenantApiUsageState(UUID tenantId) { + return DaoUtil.getData(apiUsageStateRepository.findByTenantId(tenantId)); } @Override - public void deleteUsageRecordsByTenantId(TenantId tenantId) { - usageRecordRepository.deleteUsageRecordsByTenantId(tenantId.getId()); + public void deleteApiUsageStateByTenantId(TenantId tenantId) { + apiUsageStateRepository.deleteApiUsageStateByTenantId(tenantId.getId()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index bd9ee321e0..3a0edf5eea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -39,7 +39,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.usagerecord.UsageRecordService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -74,7 +74,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private DeviceProfileService deviceProfileService; @Autowired - private UsageRecordService usageRecordService; + private ApiUsageStateService apiUsageStateService; @Autowired private EntityViewService entityViewService; @@ -121,7 +121,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe Tenant savedTenant = tenantDao.save(tenant.getId(), tenant); if (tenant.getId() == null) { deviceProfileService.createDefaultDeviceProfile(savedTenant.getId()); - usageRecordService.createDefaultUsageRecord(savedTenant.getId()); + apiUsageStateService.createDefaultApiUsageState(savedTenant.getId()); } return savedTenant; } @@ -139,7 +139,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); userService.deleteTenantAdmins(tenantId); ruleChainService.deleteRuleChainsByTenantId(tenantId); - usageRecordService.deleteUsageRecordsByTenantId(tenantId); + apiUsageStateService.deleteApiUsageStateByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); deleteEntityRelations(tenantId, tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java new file mode 100644 index 0000000000..9e543a8f7a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.usagerecord; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.tenant.TenantDao; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service +@Slf4j +public class ApiApiUsageStateServiceImpl extends AbstractEntityService implements ApiUsageStateService { + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + + private final ApiUsageStateDao apiUsageStateDao; + private final TenantDao tenantDao; + + public ApiApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao) { + this.tenantDao = tenantDao; + this.apiUsageStateDao = apiUsageStateDao; + } + + @Override + public void deleteApiUsageStateByTenantId(TenantId tenantId) { + log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + apiUsageStateDao.deleteApiUsageStateByTenantId(tenantId); + } + + @Override + public void createDefaultApiUsageState(TenantId tenantId) { + log.trace("Executing createDefaultUsageRecord [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(tenantId); + apiUsageStateValidator.validate(apiUsageState, ApiUsageState::getTenantId); + apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + } + + @Override + public ApiUsageState findTenantApiUsageState(TenantId tenantId) { + log.trace("Executing findTenantUsageRecord, tenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + return apiUsageStateDao.findTenantApiUsageState(tenantId.getId()); + } + + private DataValidator apiUsageStateValidator = + new DataValidator() { + @Override + protected void validateDataImpl(TenantId requestTenantId, ApiUsageState apiUsageState) { + if (apiUsageState.getTenantId() == null) { + throw new DataValidationException("ApiUsageState should be assigned to tenant!"); + } else { + Tenant tenant = tenantDao.findById(requestTenantId, apiUsageState.getTenantId().getId()); + if (tenant == null) { + throw new DataValidationException("Asset is referencing to non-existent tenant!"); + } + } + if (apiUsageState.getEntityId() == null) { + throw new DataValidationException("UsageRecord should be assigned to entity!"); + } else if (!EntityType.TENANT.equals(apiUsageState.getEntityId().getEntityType())) { + throw new DataValidationException("Only Tenant Usage Records are supported!"); + } else if (!apiUsageState.getTenantId().getId().equals(apiUsageState.getEntityId().getId())) { + throw new DataValidationException("Can't assign one Usage Record to multiple tenants!"); + } + } + }; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java similarity index 69% rename from dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java rename to dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java index 3b71e4ce20..1521da525e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -15,24 +15,21 @@ */ package org.thingsboard.server.dao.usagerecord; -import org.thingsboard.server.common.data.UsageRecord; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.UUID; -public interface UsageRecordDao extends Dao { +public interface ApiUsageStateDao extends Dao { /** * Save or update usage record object * - * @param usageRecord the usage record + * @param apiUsageState the usage record * @return saved usage record entity */ - UsageRecord save(TenantId tenantId, UsageRecord usageRecord); + ApiUsageState save(TenantId tenantId, ApiUsageState apiUsageState); /** * Find usage record by tenantId. @@ -40,12 +37,12 @@ public interface UsageRecordDao extends Dao { * @param tenantId the tenantId * @return the corresponding usage record */ - UsageRecord findTenantUsageRecord(UUID tenantId); + ApiUsageState findTenantApiUsageState(UUID tenantId); /** * Delete usage record by tenantId. * * @param tenantId the tenantId */ - void deleteUsageRecordsByTenantId(TenantId tenantId); + void deleteApiUsageStateByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java deleted file mode 100644 index fdaa38a1a2..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/UsageRecordServiceImpl.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.usagerecord; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.ListenableFuture; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.UsageRecord; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserCredentialsId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.exception.IncorrectParameterException; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; - -import java.util.HashMap; -import java.util.Map; - -import static org.thingsboard.server.dao.service.Validator.validateId; -import static org.thingsboard.server.dao.service.Validator.validatePageLink; -import static org.thingsboard.server.dao.service.Validator.validateString; - -@Service -@Slf4j -public class UsageRecordServiceImpl extends AbstractEntityService implements UsageRecordService { - public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - - private final UsageRecordDao usageRecordDao; - private final TenantDao tenantDao; - - public UsageRecordServiceImpl(TenantDao tenantDao, UsageRecordDao usageRecordDao) { - this.tenantDao = tenantDao; - this.usageRecordDao = usageRecordDao; - } - - @Override - public void deleteUsageRecordsByTenantId(TenantId tenantId) { - log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); - validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - usageRecordDao.deleteUsageRecordsByTenantId(tenantId); - } - - @Override - public void createDefaultUsageRecord(TenantId tenantId) { - log.trace("Executing createDefaultUsageRecord [{}]", tenantId); - validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - UsageRecord usageRecord = new UsageRecord(); - usageRecord.setTenantId(tenantId); - usageRecord.setEntityId(tenantId); - usageRecordValidator.validate(usageRecord, UsageRecord::getTenantId); - usageRecordDao.save(usageRecord.getTenantId(), usageRecord); - } - - @Override - public UsageRecord findTenantUsageRecord(TenantId tenantId) { - log.trace("Executing findTenantUsageRecord, tenantId [{}]", tenantId); - validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return usageRecordDao.findTenantUsageRecord(tenantId.getId()); - } - - private DataValidator usageRecordValidator = - new DataValidator() { - @Override - protected void validateDataImpl(TenantId requestTenantId, UsageRecord usageRecord) { - if (usageRecord.getTenantId() == null) { - throw new DataValidationException("UsageRecord should be assigned to tenant!"); - } else { - Tenant tenant = tenantDao.findById(requestTenantId, usageRecord.getTenantId().getId()); - if (tenant == null) { - throw new DataValidationException("Asset is referencing to non-existent tenant!"); - } - } - if (usageRecord.getEntityId() == null) { - throw new DataValidationException("UsageRecord should be assigned to entity!"); - } else if (!EntityType.TENANT.equals(usageRecord.getEntityId().getEntityType())) { - throw new DataValidationException("Only Tenant Usage Records are supported!"); - } else if (!usageRecord.getTenantId().getId().equals(usageRecord.getEntityId().getId())) { - throw new DataValidationException("Can't assign one Usage Record to multiple tenants!"); - } - } - }; - -} diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index e3cbbda351..c671cd2e19 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -405,11 +405,11 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); -CREATE TABLE IF NOT EXISTS usage_record ( +CREATE TABLE IF NOT EXISTS api_usage_state ( id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, created_time bigint NOT NULL, tenant_id uuid, entity_type varchar(32), entity_id uuid, - CONSTRAINT usage_record_unq_key UNIQUE (tenant_id, entity_id) + CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 6cd3adb337..d8539d9af9 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -431,13 +431,13 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) ); -CREATE TABLE IF NOT EXISTS usage_record ( +CREATE TABLE IF NOT EXISTS api_usage_state ( id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, created_time bigint NOT NULL, tenant_id uuid, entity_type varchar(32), entity_id uuid, - CONSTRAINT usage_record_unq_key UNIQUE (tenant_id, entity_id) + CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 8da1727631..2a99e868a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -27,7 +27,6 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; @@ -38,10 +37,7 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTra import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; @@ -61,7 +57,7 @@ import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.dao.usagerecord.UsageRecordService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -87,7 +83,7 @@ public abstract class AbstractServiceTest { protected UserService userService; @Autowired - protected UsageRecordService usageRecordService; + protected ApiUsageStateService apiUsageStateService; @Autowired protected AdminSettingsService adminSettingsService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java similarity index 68% rename from dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java index 28e0a6d54a..5f56506997 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseUsageRecordServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java @@ -19,17 +19,12 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.UsageRecord; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.service.AbstractServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; -public abstract class BaseUsageRecordServiceTest extends AbstractServiceTest { +public abstract class BaseApiUsageStateServiceTest extends AbstractServiceTest { private TenantId tenantId; @@ -48,9 +43,9 @@ public abstract class BaseUsageRecordServiceTest extends AbstractServiceTest { } @Test - public void testFindUsageRecordByTenantId() { - UsageRecord usageRecord = usageRecordService.findTenantUsageRecord(tenantId); - Assert.assertNotNull(usageRecord); + public void testFindApiUsageStateByTenantId() { + ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java similarity index 77% rename from dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java index 8275b92044..bbaae0a5a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/sql/UsageRecordServiceSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/sql/ApiUsageStateServiceSqlTest.java @@ -15,10 +15,9 @@ */ package org.thingsboard.server.dao.service.sql; -import org.thingsboard.server.dao.service.BaseUsageRecordServiceTest; -import org.thingsboard.server.dao.service.BaseUserServiceTest; +import org.thingsboard.server.dao.service.BaseApiUsageStateServiceTest; import org.thingsboard.server.dao.service.DaoSqlTest; @DaoSqlTest -public class UsageRecordServiceSqlTest extends BaseUsageRecordServiceTest { +public class ApiUsageStateServiceSqlTest extends BaseApiUsageStateServiceTest { } diff --git a/dao/src/test/resources/sql/hsql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql index 13c1eba943..b5629b0560 100644 --- a/dao/src/test/resources/sql/hsql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/hsql/drop-all-tables.sql @@ -27,5 +27,5 @@ DROP TABLE IF EXISTS rule_chain; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; -DROP TABLE IF EXISTS usage_record; +DROP TABLE IF EXISTS api_usage_state; DROP FUNCTION IF EXISTS to_uuid; diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index b8ab69c758..34985883d7 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -28,4 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; -DROP TABLE IF EXISTS usage_record; \ No newline at end of file +DROP TABLE IF EXISTS api_usage_state; \ No newline at end of file diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index 4333bc33ee..211cebb2ad 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -28,4 +28,4 @@ DROP TABLE IF EXISTS tb_schema_settings; DROP TABLE IF EXISTS oauth2_client_registration; DROP TABLE IF EXISTS oauth2_client_registration_info; DROP TABLE IF EXISTS oauth2_client_registration_template; -DROP TABLE IF EXISTS usage_record; \ No newline at end of file +DROP TABLE IF EXISTS api_usage_state; \ No newline at end of file From ef29aba24c935dc963382a5103e6c04944f6a486 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 20 Oct 2020 10:11:26 +0300 Subject: [PATCH 05/30] created usage stats queue consumer and producer --- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbUsageStatsService.java | 15 ++++++ .../usagestats/TbUsageStatsService.java | 15 ++++++ .../src/main/resources/thingsboard.yml | 1 + .../provider/AwsSqsMonolithQueueFactory.java | 45 ++++++++++------ .../provider/AwsSqsTbCoreQueueFactory.java | 12 +++++ .../AwsSqsTbRuleEngineQueueFactory.java | 6 +++ .../provider/AwsSqsTransportQueueFactory.java | 6 +++ .../InMemoryMonolithQueueFactory.java | 12 +++++ .../InMemoryTbTransportQueueFactory.java | 7 +++ .../provider/KafkaMonolithQueueFactory.java | 23 ++++++++ .../provider/KafkaTbCoreQueueFactory.java | 23 ++++++++ .../KafkaTbRuleEngineQueueFactory.java | 11 ++++ .../KafkaTbTransportQueueFactory.java | 11 ++++ .../provider/PubSubMonolithQueueFactory.java | 12 +++++ .../provider/PubSubTbCoreQueueFactory.java | 12 +++++ .../PubSubTbRuleEngineQueueFactory.java | 6 +++ .../provider/PubSubTransportQueueFactory.java | 6 +++ .../RabbitMqMonolithQueueFactory.java | 52 +++++++++++++------ .../provider/RabbitMqTbCoreQueueFactory.java | 12 +++++ .../RabbitMqTbRuleEngineQueueFactory.java | 6 +++ .../RabbitMqTransportQueueFactory.java | 6 +++ .../ServiceBusMonolithQueueFactory.java | 12 +++++ .../ServiceBusTbCoreQueueFactory.java | 23 ++++++-- .../ServiceBusTbRuleEngineQueueFactory.java | 17 ++++-- .../ServiceBusTransportQueueFactory.java | 36 ++++++++----- .../queue/provider/TbCoreQueueFactory.java | 7 +-- .../TbUsageStatsClientQueueFactory.java | 5 +- .../queue/settings/TbQueueCoreSettings.java | 3 ++ 29 files changed, 337 insertions(+), 67 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3f03b65416..e806eccf66 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java index 6611507531..bf20419670 100644 --- a/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/usagestats/DefaultTbUsageStatsService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.usagestats; import org.springframework.stereotype.Service; diff --git a/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java index 35a7051896..e735c5e2e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/usagestats/TbUsageStatsService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.usagestats; import org.thingsboard.server.common.msg.queue.TbCallback; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index deb28ffd99..f0f4b9dd54 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -673,6 +673,7 @@ queue: poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index c1d724c207..0299720005 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -22,7 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 4889a7182d..4625b9d4fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index cf43bc5fe4..851b8cc40e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory notificationAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 8106ee084b..103711f36b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 8ebda6e7b9..f7a47ecf3a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); } + + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 9e8e326f0d..9ddf72534c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory public TbQueueConsumer> createTransportNotificationsConsumer() { return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 31ef4d3e45..538d91aa0f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index bf2ec6751e..5e1b398e91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getUsageStatsTopic()); + consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index dc44caf3f0..118f0f3316 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 1cf8bca4b2..7961bb97bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { return responseBuilder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 29a60af1e2..d7c7542626 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 7d85b5c3ba..71a1ceaf9b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index a6105fea0c..ad679039e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -146,6 +147,11 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 72665c5163..efde055271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -114,6 +115,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1f812ea13c..e6eff652be 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -22,7 +22,14 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -91,64 +98,64 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @@ -175,6 +182,17 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 65d582c4bc..e3064dd62d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -163,6 +164,17 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 26abf78164..a1097049db 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -144,6 +145,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index 03cd6eb557..b28a2cb7e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -111,6 +112,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3db6496b7d..a205f9b061 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -180,6 +181,17 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index a3ba577f06..f00aee331b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -101,7 +103,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -111,7 +113,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -122,10 +124,10 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -162,6 +164,17 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index e4f6dbfbb4..40a183a254 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -93,7 +95,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @@ -103,7 +105,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @@ -114,10 +116,10 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @@ -143,6 +145,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact return builder.build(); } + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index 1e900f37e5..4781d46583 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -18,19 +18,24 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -69,17 +74,17 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumerTemplate = + TbQueueConsumer> consumerTemplate = new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); @@ -90,20 +95,25 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index d009757997..7c19a1ba2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,12 +16,12 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -83,10 +83,7 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * * @return */ - default TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - //TODO: implement - return null; - } + TbQueueConsumer> createToUsageStatsServiceMsgConsumer(); /** * Used to consume high priority messages by TB Core Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java index 1811877338..f2ce1a9fe5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java @@ -21,9 +21,6 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; public interface TbUsageStatsClientQueueFactory { - default TbQueueProducer> createToUsageStatsServiceMsgProducer() { - //TODO: implement - return null; - } + TbQueueProducer> createToUsageStatsServiceMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 1c260c754d..d406139dbb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -26,6 +26,9 @@ public class TbQueueCoreSettings { @Value("${queue.core.topic}") private String topic; + @Value("${queue.core.usage-stats-topic}") + private String usageStatsTopic; + @Value("${queue.core.partitions}") private int partitions; } From cc2446442d7423a7038cb3a95e239e0a4e7c276d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 20 Oct 2020 12:00:35 +0300 Subject: [PATCH 06/30] Api Usage Stats flow --- .../DefaultTbApiUsageStateService.java | 105 ++++++++++++++++++ .../apiusage/DefaultTbUsageStatsService.java | 30 ----- ...rvice.java => TbApiUsageStateService.java} | 2 +- .../service/apiusage/TenantApiUsageState.java | 47 ++++++++ .../queue/DefaultTbCoreConsumerService.java | 6 +- .../usagestats/DefaultTbUsageStatsClient.java | 47 ++++---- .../service/DefaultTransportService.java | 36 +++++- 7 files changed, 217 insertions(+), 56 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java rename application/src/main/java/org/thingsboard/server/service/apiusage/{TbUsageStatsService.java => TbApiUsageStateService.java} (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java new file mode 100644 index 0000000000..8a50c9d62b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; + +@Slf4j +@TbCoreComponent +@Service +public class DefaultTbApiUsageStateService implements TbApiUsageStateService { + + private final ApiUsageStateService apiUsageStateService; + private final TimeseriesService tsService; + private final ZoneId zoneId; + private final Map tenantStates = new ConcurrentHashMap<>(); + + public DefaultTbApiUsageStateService(ApiUsageStateService apiUsageStateService, TimeseriesService tsService) { + this.apiUsageStateService = apiUsageStateService; + this.tsService = tsService; + this.zoneId = SchedulerUtils.getZoneId("UTC"); + } + + @Override + public void process(TbProtoQueueMsg msg, TbCallback callback) { + ToUsageStatsServiceMsg statsMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); + TenantApiUsageState tenantState = getOrFetchState(tenantId); + long ts = tenantState.getCurrentMonthTs(); + List updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); + for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { + ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); + long newValue = tenantState.add(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); + } + tsService.save(tenantId, tenantState.getEntityId(), updatedEntries, 0L); + } + + private TenantApiUsageState getOrFetchState(TenantId tenantId) { + TenantApiUsageState tenantState = tenantStates.get(tenantId); + if (tenantState == null) { + long currentMonthTs = LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli(); + ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + tenantState = new TenantApiUsageState(currentMonthTs, dbStateEntity.getEntityId()); + try { + List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getEntityId()).get(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + TsKvEntry keyEntry = null; + for (TsKvEntry tsKvEntry : dbValues) { + if (tsKvEntry.getKey().equals(key.name())) { + keyEntry = tsKvEntry; + break; + } + } + if (keyEntry != null) { + tenantState.put(key, keyEntry.getLongValue().get()); + } else { + tenantState.put(key, 0L); + } + } + tenantStates.put(tenantId, tenantState); + } catch (InterruptedException | ExecutionException e) { + log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); + } + } + return tenantState; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java deleted file mode 100644 index e2679d6433..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbUsageStatsService.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.apiusage; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; - -@Service -public class DefaultTbUsageStatsService implements TbUsageStatsService { - @Override - public void process(TbProtoQueueMsg msg, TbCallback callback) { - - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java rename to application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index 3c54479eaf..b5aeedeb06 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbUsageStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbUsageStatsService { +public interface TbApiUsageStateService { void process(TbProtoQueueMsg msg, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java new file mode 100644 index 0000000000..a8be19914b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.Getter; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class TenantApiUsageState { + + private final Map values = new ConcurrentHashMap<>(); + @Getter + private final EntityId entityId; + @Getter + private volatile long currentMonthTs; + + public TenantApiUsageState(long currentMonthTs, EntityId entityId) { + this.entityId = entityId; + this.currentMonthTs = currentMonthTs; + } + + public void put(ApiUsageRecordKey key, Long value) { + values.put(key, value); + } + + public long add(ApiUsageRecordKey key, long value) { + long result = values.getOrDefault(key, 0L) + value; + values.put(key, result); + return result; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a389eefd1a..51fce853db 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -62,7 +62,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.service.apiusage.TbUsageStatsService; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -92,7 +92,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> mainConsumer; private final DeviceStateService stateService; - private final TbUsageStatsService statsService; + private final TbApiUsageStateService statsService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; @@ -106,7 +106,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService[] values = new ConcurrentMap[ApiUsageRecordKey.values().length]; @@ -69,28 +71,33 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { } private void reportStats() { - ConcurrentMap report = new ConcurrentHashMap<>(); + try { + ConcurrentMap report = new ConcurrentHashMap<>(); - for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { - values[key.ordinal()].forEach(((tenantId, atomicLong) -> { - long value = atomicLong.getAndSet(0); - if (value > 0) { - ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> { - ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder(); - msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - return msg; - }); - msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build()); - } + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + values[key.ordinal()].forEach(((tenantId, atomicLong) -> { + long value = atomicLong.getAndSet(0); + if (value > 0) { + ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> { + ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder(); + msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + return msg; + }); + msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build()); + } + })); + } + + report.forEach(((tenantId, builder) -> { + //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId); + msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); })); + log.info("Report statistics for: {} tenants", report.size()); + } catch (Exception e) { + log.warn("Failed to report statistics: ", e); } - - report.forEach(((tenantId, builder) -> { - //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId); - msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); - })); } @Override diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 774a437e39..010a3cd899 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; @@ -78,6 +79,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; import org.thingsboard.server.queue.util.TbTransportComponent; import javax.annotation.PostConstruct; @@ -122,6 +124,7 @@ public class DefaultTransportService implements TransportService { private final StatsFactory statsFactory; private final TransportDeviceProfileCache deviceProfileCache; private final TransportTenantProfileCache tenantProfileCache; + private final TbUsageStatsClient apiUsageStatsClient; private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; private final SchedulerComponent scheduler; @@ -150,7 +153,8 @@ public class DefaultTransportService implements TransportService { StatsFactory statsFactory, TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, - TransportRateLimitService rateLimitService, DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { + TbUsageStatsClient apiUsageStatsClient, TransportRateLimitService rateLimitService, + DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; @@ -158,6 +162,7 @@ public class DefaultTransportService implements TransportService { this.statsFactory = statsFactory; this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; + this.apiUsageStatsClient = apiUsageStatsClient; this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; this.scheduler = scheduler; @@ -362,7 +367,7 @@ public class DefaultTransportService implements TransportService { reportActivityInternal(sessionInfo); TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), new ApiStatsProxyCallback<>(tenantId, dataPoints, callback)); for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("deviceName", sessionInfo.getDeviceName()); @@ -806,4 +811,31 @@ public class DefaultTransportService implements TransportService { callback.onError(t); } } + + private class ApiStatsProxyCallback implements TransportServiceCallback { + private final TenantId tenantId; + private final int dataPoints; + private final TransportServiceCallback callback; + + public ApiStatsProxyCallback(TenantId tenantId, int dataPoints, TransportServiceCallback callback) { + this.tenantId = tenantId; + this.dataPoints = dataPoints; + this.callback = callback; + } + + @Override + public void onSuccess(T msg) { + try { + apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.MSG_COUNT, 1); + apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.DP_TRANSPORT_COUNT, dataPoints); + } finally { + callback.onSuccess(msg); + } + } + + @Override + public void onError(Throwable e) { + callback.onError(e); + } + } } From 70fe28a5ac172bbc7aa093fdee04e8151118485e Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 20 Oct 2020 13:59:14 +0300 Subject: [PATCH 07/30] added usage stats consumer and producer to queue providers --- .../queue/DefaultTbCoreConsumerService.java | 20 +++++++---- .../common/msg/queue/TopicPartitionInfo.java | 4 +++ .../provider/TbCoreQueueProducerProvider.java | 34 ++++++++++++------ .../provider/TbQueueProducerProvider.java | 8 ++--- .../TbRuleEngineProducerProvider.java | 35 ++++++++++++------- .../TbTransportQueueProducerProvider.java | 25 +++++++------ .../usagestats/DefaultTbUsageStatsClient.java | 2 +- 7 files changed, 81 insertions(+), 47 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 51fce853db..929d5000f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -30,28 +30,29 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.dao.util.mapping.JacksonUtil; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto; -import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; @@ -62,7 +63,6 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -143,6 +143,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService tpi.newByTopic(usageStatsConsumer.getTopic())) + .collect(Collectors.toSet())); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index bf0fffe404..c2dec02612 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -50,6 +50,10 @@ public class TopicPartitionInfo { this.fullTopicName = tmp; } + public TopicPartitionInfo newByTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + } + public String getTopic() { return topic; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9fe5de0bcc..e620259960 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,11 +33,12 @@ import javax.annotation.PostConstruct; public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private final TbCoreQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -45,30 +51,36 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index fa20f11960..8c3a0b6565 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -69,9 +69,5 @@ public interface TbQueueProducerProvider { * * @return */ - default TbQueueProducer> getTbUsageStatsMsgProducer() { - //TODO: implement - return null; - } - + TbQueueProducer> getTbUsageStatsMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 69de7ca3e2..309def1269 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,12 +33,12 @@ import javax.annotation.PostConstruct; public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private final TbRuleEngineQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toRuleEngineNotifications; - private TbQueueProducer> toTbCoreNotifications; - + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -46,30 +51,36 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return toTransport; } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { return toRuleEngineNotifications; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { return toTbCoreNotifications; } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 9d11aa727b..dee6508ef4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,9 +33,9 @@ import javax.annotation.PostConstruct; public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toRuleEngine; - private TbQueueProducer> toTbCore; - private TbQueueProducer> toUsageStats; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toUsageStats; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -44,32 +49,32 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> getRuleEngineMsgProducer() { return toRuleEngine; } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> getTbCoreMsgProducer() { return toTbCore; } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override - public TbQueueProducer> getTbUsageStatsMsgProducer() { + public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java index 35bb0c2a6a..5a94cd505e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java @@ -91,7 +91,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { report.forEach(((tenantId, builder) -> { //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages? - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).newByTopic(msgProducer.getDefaultTopic()); msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null); })); log.info("Report statistics for: {} tenants", report.size()); From 9ec4b776728a8da85bc552a5eefb448133ae83bc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 20 Oct 2020 15:50:04 +0300 Subject: [PATCH 08/30] Improvements to ApiUsageState persistence --- .../DefaultTbApiUsageStateService.java | 119 ++++++++++++++---- .../apiusage/TbApiUsageStateService.java | 12 +- .../service/apiusage/TenantApiUsageState.java | 49 ++++++-- .../dao/usagerecord/ApiUsageStateService.java | 2 +- .../data/plugin/ComponentLifecycleEvent.java | 2 +- .../common/msg/tools/SchedulerUtils.java | 48 +++++++ .../ApiApiUsageStateServiceImpl.java | 4 +- 7 files changed, 202 insertions(+), 34 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 8a50c9d62b..1b6cdf3cff 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -16,9 +16,11 @@ package org.thingsboard.server.service.apiusage; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; @@ -30,69 +32,144 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.profile.TbTenantProfileCache; -import java.time.LocalDate; -import java.time.ZoneId; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; @Slf4j @TbCoreComponent @Service public class DefaultTbApiUsageStateService implements TbApiUsageStateService { + public static final String HOURLY = "HOURLY_"; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; - private final ZoneId zoneId; + private final SchedulerComponent scheduler; + private final TbTenantProfileCache tenantProfileCache; private final Map tenantStates = new ConcurrentHashMap<>(); - public DefaultTbApiUsageStateService(ApiUsageStateService apiUsageStateService, TimeseriesService tsService) { + @Value("${usage.stats.report.enabled:true}") + private boolean enabled; + + @Value("${usage.stats.check.cycle:60000}") + private long nextCycleCheckInterval; + + private final Lock updateLock = new ReentrantLock(); + + public DefaultTbApiUsageStateService(ApiUsageStateService apiUsageStateService, TimeseriesService tsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; - this.zoneId = SchedulerUtils.getZoneId("UTC"); + this.scheduler = scheduler; + this.tenantProfileCache = tenantProfileCache; + } + + @PostConstruct + public void init() { + if (enabled) { + scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS); + } } @Override public void process(TbProtoQueueMsg msg, TbCallback callback) { ToUsageStatsServiceMsg statsMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); - TenantApiUsageState tenantState = getOrFetchState(tenantId); - long ts = tenantState.getCurrentMonthTs(); - List updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); - for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { - ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); - long newValue = tenantState.add(recordKey, kvProto.getValue()); - updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); + TenantApiUsageState tenantState; + List updatedEntries; + updateLock.lock(); + try { + tenantState = getOrFetchState(tenantId); + long ts = tenantState.getCurrentCycleTs(); + long hourTs = tenantState.getCurrentHourTs(); + long newHourTs = SchedulerUtils.getStartOfCurrentHour(); + if (newHourTs != hourTs) { + tenantState.setHour(newHourTs); + } + updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); + for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { + ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); + long newValue = tenantState.add(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); + newValue = tenantState.addToHourly(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(HOURLY + recordKey.name(), newValue))); + } + } finally { + updateLock.unlock(); } tsService.save(tenantId, tenantState.getEntityId(), updatedEntries, 0L); } + @Override + public TenantApiUsageState getApiUsageState(TenantId tenantId) { + return null; + } + + @Override + public void onAddedToAllowList(TenantId tenantId) { + + } + + @Override + public void onAddedToDenyList(TenantId tenantId) { + + } + + private void checkStartOfNextCycle() { + updateLock.lock(); + try { + long now = System.currentTimeMillis(); + tenantStates.values().forEach(state -> { + if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) { + state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth()); + } + }); + } finally { + updateLock.unlock(); + } + } + private TenantApiUsageState getOrFetchState(TenantId tenantId) { TenantApiUsageState tenantState = tenantStates.get(tenantId); if (tenantState == null) { - long currentMonthTs = LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli(); ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); - tenantState = new TenantApiUsageState(currentMonthTs, dbStateEntity.getEntityId()); + if (dbStateEntity == null) { + try { + dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId); + } catch (Exception e) { + dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); + } + } + tenantState = new TenantApiUsageState(dbStateEntity.getEntityId()); try { List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getEntityId()).get(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { - TsKvEntry keyEntry = null; + boolean cycleEntryFound = false; + boolean hourlyEntryFound = false; for (TsKvEntry tsKvEntry : dbValues) { if (tsKvEntry.getKey().equals(key.name())) { - keyEntry = tsKvEntry; + cycleEntryFound = true; + tenantState.put(key, tsKvEntry.getLongValue().get()); + } else if (tsKvEntry.getKey().equals(HOURLY + key.name())) { + hourlyEntryFound = true; + if (tsKvEntry.getTs() == tenantState.getCurrentHourTs()) { + tenantState.putHourly(key, tsKvEntry.getLongValue().get()); + } + } + if (cycleEntryFound && hourlyEntryFound) { break; } } - if (keyEntry != null) { - tenantState.put(key, keyEntry.getLongValue().get()); - } else { - tenantState.put(key, 0L); - } } tenantStates.put(tenantId, tenantState); } catch (InterruptedException | ExecutionException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index b5aeedeb06..e79b79d069 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -15,11 +15,19 @@ */ package org.thingsboard.server.service.apiusage; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; public interface TbApiUsageStateService { - void process(TbProtoQueueMsg msg, TbCallback callback); + void process(TbProtoQueueMsg msg, TbCallback callback); + + TenantApiUsageState getApiUsageState(TenantId tenantId); + + void onAddedToAllowList(TenantId tenantId); + + void onAddedToDenyList(TenantId tenantId); + } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index a8be19914b..29335c3326 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -18,30 +18,65 @@ package org.thingsboard.server.service.apiusage; import lombok.Getter; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public class TenantApiUsageState { - private final Map values = new ConcurrentHashMap<>(); + private final Map currentCycleValues = new ConcurrentHashMap<>(); + private final Map currentHourValues = new ConcurrentHashMap<>(); + @Getter private final EntityId entityId; @Getter - private volatile long currentMonthTs; + private volatile long currentCycleTs; + @Getter + private volatile long nextCycleTs; + @Getter + private volatile long currentHourTs; - public TenantApiUsageState(long currentMonthTs, EntityId entityId) { + public TenantApiUsageState(EntityId entityId) { this.entityId = entityId; - this.currentMonthTs = currentMonthTs; + this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); + this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); + this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); } public void put(ApiUsageRecordKey key, Long value) { - values.put(key, value); + currentCycleValues.put(key, value); + } + + public void putHourly(ApiUsageRecordKey key, Long value) { + currentHourValues.put(key, value); } public long add(ApiUsageRecordKey key, long value) { - long result = values.getOrDefault(key, 0L) + value; - values.put(key, result); + long result = currentCycleValues.getOrDefault(key, 0L) + value; + currentCycleValues.put(key, result); return result; } + + public long addToHourly(ApiUsageRecordKey key, long value) { + long result = currentHourValues.getOrDefault(key, 0L) + value; + currentHourValues.put(key, result); + return result; + } + + public void setHour(long currentHourTs) { + this.currentHourTs = currentHourTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentHourValues.put(key, 0L); + } + } + + public void setCycles(long currentCycleTs, long nextCycleTs) { + this.currentCycleTs = currentCycleTs; + this.nextCycleTs = nextCycleTs; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + currentCycleValues.put(key, 0L); + } + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index ad6c73f63e..c39e2a69dc 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -24,5 +24,5 @@ public interface ApiUsageStateService { void deleteApiUsageStateByTenantId(TenantId tenantId); - void createDefaultApiUsageState(TenantId id); + ApiUsageState createDefaultApiUsageState(TenantId id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index 87adc31713..1f242a0f88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -21,5 +21,5 @@ import java.io.Serializable; * @author Andrew Shvayka */ public enum ComponentLifecycleEvent implements Serializable { - CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED + CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, ADDED_TO_ALLOW_LIST, ADDED_TO_DENY_LIST } \ No newline at end of file diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java index fea6fcb337..421066b30b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java @@ -15,16 +15,64 @@ */ package org.thingsboard.server.common.msg.tools; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjuster; +import java.time.temporal.TemporalAdjusters; +import java.time.temporal.TemporalUnit; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoUnit.MONTHS; + public class SchedulerUtils { + private final static ZoneId UTC = ZoneId.of("UTC"); private static final ConcurrentMap tzMap = new ConcurrentHashMap<>(); public static ZoneId getZoneId(String tz) { return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); } + public static long getStartOfCurrentHour() { + return getStartOfCurrentHour(UTC); + } + + public static long getStartOfCurrentHour(ZoneId zoneId) { + return LocalDateTime.now(ZoneOffset.UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli(); + } + + public static long getStartOfCurrentMonth() { + return getStartOfCurrentMonth(UTC); + } + + public static long getStartOfCurrentMonth(ZoneId zoneId) { + return LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextMonth() { + return getStartOfNextMonth(UTC); + } + + public static long getStartOfNextMonth(ZoneId zoneId) { + return LocalDate.now().with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static long getStartOfNextNextMonth() { + return getStartOfNextNextMonth(UTC); + } + + public static long getStartOfNextNextMonth(ZoneId zoneId) { + return LocalDate.now().with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli(); + } + + public static TemporalAdjuster firstDayOfNextNextMonth() { + return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java index 9e543a8f7a..6be56a1815 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java @@ -49,14 +49,14 @@ public class ApiApiUsageStateServiceImpl extends AbstractEntityService implement } @Override - public void createDefaultApiUsageState(TenantId tenantId) { + public ApiUsageState createDefaultApiUsageState(TenantId tenantId) { log.trace("Executing createDefaultUsageRecord [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); ApiUsageState apiUsageState = new ApiUsageState(); apiUsageState.setTenantId(tenantId); apiUsageState.setEntityId(tenantId); apiUsageStateValidator.validate(apiUsageState, ApiUsageState::getTenantId); - apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + return apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); } @Override From 7e8837603016db5c3ab79bd0408f6b7df14710db Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 20 Oct 2020 18:55:52 +0300 Subject: [PATCH 09/30] created alias ApiUsageState --- .../DefaultTbApiUsageStateService.java | 6 ++-- .../service/apiusage/TenantApiUsageState.java | 7 +++-- .../queue/DefaultTbCoreConsumerService.java | 3 ++ .../service/security/AccessValidator.java | 27 ++++++++++++++++++ .../service/security/permission/Resource.java | 3 +- .../permission/TenantAdminPermissions.java | 1 + .../dao/usagerecord/ApiUsageStateService.java | 3 ++ .../common/data/id/EntityIdFactory.java | 2 ++ .../data/query/ApiUsageStateFilter.java | 28 +++++++++++++++++++ .../common/data/query/EntityFilter.java | 1 + .../common/data/query/EntityFilterType.java | 3 +- .../query/DefaultEntityQueryRepository.java | 4 +++ .../ApiApiUsageStateServiceImpl.java | 8 ++++++ ui-ngx/src/app/core/http/entity.service.ts | 3 ++ ui-ngx/src/app/shared/models/alias.models.ts | 9 +++++- .../assets/locale/locale.constant-en_US.json | 1 + 16 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 1b6cdf3cff..185637e543 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -62,7 +62,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { @Value("${usage.stats.report.enabled:true}") private boolean enabled; - @Value("${usage.stats.check.cycle:60000}") + @Value("${usage.stats.check.cycle:6000}") private long nextCycleCheckInterval; private final Lock updateLock = new ReentrantLock(); @@ -107,7 +107,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } finally { updateLock.unlock(); } - tsService.save(tenantId, tenantState.getEntityId(), updatedEntries, 0L); + tsService.save(tenantId, tenantState.getId(), updatedEntries, 0L); } @Override @@ -150,7 +150,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); } } - tenantState = new TenantApiUsageState(dbStateEntity.getEntityId()); + tenantState = new TenantApiUsageState(dbStateEntity.getId()); try { List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getEntityId()).get(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 29335c3326..2c483cea29 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.apiusage; import lombok.Getter; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.tools.SchedulerUtils; @@ -29,7 +30,7 @@ public class TenantApiUsageState { private final Map currentHourValues = new ConcurrentHashMap<>(); @Getter - private final EntityId entityId; + private final ApiUsageStateId id; @Getter private volatile long currentCycleTs; @Getter @@ -37,8 +38,8 @@ public class TenantApiUsageState { @Getter private volatile long currentHourTs; - public TenantApiUsageState(EntityId entityId) { - this.entityId = entityId; + public TenantApiUsageState(ApiUsageStateId id) { + this.id = id; this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 929d5000f0..ee2aa71e73 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -396,6 +396,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService callback) { + if (currentUser.isSystemAdmin()) { + callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); + } else { + ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId())); + if (apiUsageState == null) { + callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!")); + } else { + try { + accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState); + } catch (ThingsboardException e) { + callback.onSuccess(ValidationResult.accessDenied(e.getMessage())); + } + callback.onSuccess(ValidationResult.ok(apiUsageState)); + } + } + } + private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback callback) { if (currentUser.isSystemAdmin()) { callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 08534dc2d4..9d16760464 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -35,7 +35,8 @@ public enum Resource { OAUTH2_CONFIGURATION_INFO(), OAUTH2_CONFIGURATION_TEMPLATE(), TENANT_PROFILE(EntityType.TENANT_PROFILE), - DEVICE_PROFILE(EntityType.DEVICE_PROFILE); + DEVICE_PROFILE(EntityType.DEVICE_PROFILE), + API_USAGE_STATE(EntityType.API_USAGE_STATE); private final EntityType entityType; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 4124292c95..e0c34200f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index c39e2a69dc..8131291fbb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.usagerecord; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.TenantId; public interface ApiUsageStateService { @@ -25,4 +26,6 @@ public interface ApiUsageStateService { void deleteApiUsageStateByTenantId(TenantId tenantId); ApiUsageState createDefaultApiUsageState(TenantId id); + + ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index dc1589814f..8a1a769c5e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -66,6 +66,8 @@ public class EntityIdFactory { return new DeviceProfileId(uuid); case TENANT_PROFILE: return new TenantProfileId(uuid); + case API_USAGE_STATE: + return new ApiUsageStateId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java new file mode 100644 index 0000000000..436aa44230 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class ApiUsageStateFilter implements EntityFilter { + @Override + public EntityFilterType getType() { + return EntityFilterType.API_USAGE_STATE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index d3b9c39530..f2a6dde626 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), + @JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"), @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java index 0c05fdc8e4..12860b8453 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java @@ -25,7 +25,8 @@ public enum EntityFilterType { RELATIONS_QUERY("relationsQuery"), ASSET_SEARCH_QUERY("assetSearchQuery"), DEVICE_SEARCH_QUERY("deviceSearchQuery"), - ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); + ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"), + API_USAGE_STATE("apiUsageState"); private final String label; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 75f7658a5a..380e166aa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -210,6 +210,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.CUSTOMER, "customer"); entityTableMap.put(EntityType.USER, "tb_user"); entityTableMap.put(EntityType.TENANT, "tenant"); + entityTableMap.put(EntityType.API_USAGE_STATE, "(select aus.id, aus.created_time, aus.tenant_id, '' as name, '' as additional_info from api_usage_state as aus)"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ @@ -431,6 +432,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { case DEVICE_SEARCH_QUERY: case ASSET_SEARCH_QUERY: case ENTITY_VIEW_SEARCH_QUERY: + case API_USAGE_STATE: return ""; default: throw new RuntimeException("Not implemented!"); @@ -682,6 +684,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return EntityType.ENTITY_VIEW; case RELATIONS_QUERY: return ((RelationsQueryFilter) entityFilter).getRootEntity().getEntityType(); + case API_USAGE_STATE: + return EntityType.API_USAGE_STATE; default: throw new RuntimeException("Not implemented!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java index 6be56a1815..00ecde9d95 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -66,6 +67,13 @@ public class ApiApiUsageStateServiceImpl extends AbstractEntityService implement return apiUsageStateDao.findTenantApiUsageState(tenantId.getId()); } + @Override + public ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id) { + log.trace("Executing findApiUsageStateById, tenantId [{}], apiUsageStateId [{}]", tenantId, id); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(id, "Incorrect apiUsageStateId " + id); + return apiUsageStateDao.findById(tenantId, id.getId()); } + private DataValidator apiUsageStateValidator = new DataValidator() { @Override diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index a510735855..acd89e3c1b 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -753,6 +753,9 @@ export class EntityService { case AliasFilterType.entityViewType: result.entityFilter = deepClone(filter); return of(result); + case AliasFilterType.apiUsageState: + result.entityFilter = deepClone(filter); + return of(result); case AliasFilterType.relationsQuery: result.stateEntity = filter.rootStateEntity; let rootEntityType; diff --git a/ui-ngx/src/app/shared/models/alias.models.ts b/ui-ngx/src/app/shared/models/alias.models.ts index 35b52435a2..f437931f27 100644 --- a/ui-ngx/src/app/shared/models/alias.models.ts +++ b/ui-ngx/src/app/shared/models/alias.models.ts @@ -28,6 +28,7 @@ export enum AliasFilterType { assetType = 'assetType', deviceType = 'deviceType', entityViewType = 'entityViewType', + apiUsageState = 'apiUsageState', relationsQuery = 'relationsQuery', assetSearchQuery = 'assetSearchQuery', deviceSearchQuery = 'deviceSearchQuery', @@ -43,6 +44,7 @@ export const aliasFilterTypeTranslationMap = new Map( [ AliasFilterType.assetType, 'alias.filter-type-asset-type' ], [ AliasFilterType.deviceType, 'alias.filter-type-device-type' ], [ AliasFilterType.entityViewType, 'alias.filter-type-entity-view-type' ], + [ AliasFilterType.apiUsageState, 'alias.filter-type-apiUsageState' ], [ AliasFilterType.relationsQuery, 'alias.filter-type-relations-query' ], [ AliasFilterType.assetSearchQuery, 'alias.filter-type-asset-search-query' ], [ AliasFilterType.deviceSearchQuery, 'alias.filter-type-device-search-query' ], @@ -106,6 +108,10 @@ export interface EntitySearchQueryFilter { fetchLastLevelOnly?: boolean; } +export interface ApiUsageStateFilter { + +} + export interface AssetSearchQueryFilter extends EntitySearchQueryFilter { assetTypes?: string[]; } @@ -129,7 +135,8 @@ export type EntityFilters = RelationsQueryFilter & AssetSearchQueryFilter & DeviceSearchQueryFilter & - EntityViewSearchQueryFilter; + EntityViewSearchQueryFilter & + EntitySearchQueryFilter; export interface EntityAliasFilter extends EntityFilters { type?: AliasFilterType; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 453d5d2abd..082c2241b4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -288,6 +288,7 @@ "filter-type-device-search-query-description": "Devices with types {{deviceTypes}} that have {{relationType}} relation {{direction}} {{rootEntity}}", "filter-type-entity-view-search-query": "Entity view search query", "filter-type-entity-view-search-query-description": "Entity views with types {{entityViewTypes}} that have {{relationType}} relation {{direction}} {{rootEntity}}", + "filter-type-apiUsageState": "Api Usage State", "entity-filter": "Entity filter", "resolve-multiple": "Resolve as multiple entities", "filter-type": "Filter type", From f7aa8df607f494444ec0f8261a37721d5335f8ba Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 20 Oct 2020 18:57:08 +0300 Subject: [PATCH 10/30] refactored --- .../server/service/apiusage/DefaultTbApiUsageStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 185637e543..1a7d1cfae1 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -62,7 +62,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { @Value("${usage.stats.report.enabled:true}") private boolean enabled; - @Value("${usage.stats.check.cycle:6000}") + @Value("${usage.stats.check.cycle:60000}") private long nextCycleCheckInterval; private final Lock updateLock = new ReentrantLock(); From 9d14a38966c5351edd16c01b339a9ff3c19e88e7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 20 Oct 2020 18:57:15 +0300 Subject: [PATCH 11/30] Tenant Profile updates for ApiUsageStateService --- .../server/actors/app/AppActor.java | 9 +- .../DefaultTbApiUsageStateService.java | 37 +++++- .../apiusage/TbApiUsageStateService.java | 6 +- .../service/apiusage/TenantApiUsageState.java | 122 +++++++++++++++++- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 4 +- .../processing/AbstractConsumerService.java | 18 ++- .../dao/usagerecord/ApiUsageStateService.java | 5 +- .../server/common/data/ApiUsageState.java | 30 ++--- .../ApiApiUsageStateServiceImpl.java | 8 ++ 10 files changed, 197 insertions(+), 44 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 826596db56..fea5558540 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -40,8 +40,6 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; @@ -150,15 +148,12 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { - if (msg.getEntityId().getEntityType() == EntityType.TENANT_PROFILE) { - tenantProfileCache.evict(new TenantProfileId(msg.getEntityId().getId())); - } else { + if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) { log.warn("Message has system tenant id: {}", msg); } } else { - if (msg.getEntityId().getEntityType() == EntityType.TENANT) { + if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) { TenantId tenantId = new TenantId(msg.getEntityId().getId()); - tenantProfileCache.evict(tenantId); if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); deletedTenants.add(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 1b6cdf3cff..84801224e1 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -87,6 +88,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); TenantApiUsageState tenantState; List updatedEntries; + boolean stateUpdated = false; updateLock.lock(); try { tenantState = getOrFetchState(tenantId); @@ -101,13 +103,21 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); long newValue = tenantState.add(recordKey, kvProto.getValue()); updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); - newValue = tenantState.addToHourly(recordKey, kvProto.getValue()); - updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(HOURLY + recordKey.name(), newValue))); + long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); + updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(HOURLY + recordKey.name(), newHourlyValue))); + stateUpdated |= tenantState.checkStateUpdatedDueToThreshold(recordKey); } } finally { updateLock.unlock(); } tsService.save(tenantId, tenantState.getEntityId(), updatedEntries, 0L); + if (stateUpdated) { + // Save new state into the database; + apiUsageStateService.update(tenantState.getApiUsageState()); + //TODO: clear cache on cluster repartition. + //TODO: update profiles on tenant and profile updates. + //TODO: broadcast to everyone notifications about enabled/disabled features. + } } @Override @@ -116,12 +126,26 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } @Override - public void onAddedToAllowList(TenantId tenantId) { - + public void onTenantProfileUpdate(TenantProfileId tenantProfileId) { + TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); + updateLock.lock(); + try { + tenantStates.values().forEach(state -> { + if (tenantProfile.getId().equals(state.getTenantProfileId())) { + state.setTenantProfileData(tenantProfile.getProfileData()); + if (state.checkStateUpdatedDueToThresholds()) { + apiUsageStateService.update(state.getApiUsageState()); + //TODO: send notification to cluster; + } + } + }); + } finally { + updateLock.unlock(); + } } @Override - public void onAddedToDenyList(TenantId tenantId) { + public void onTenantUpdate(TenantId tenantId) { } @@ -150,7 +174,8 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); } } - tenantState = new TenantApiUsageState(dbStateEntity.getEntityId()); + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity); try { List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getEntityId()).get(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index e79b79d069..d17bd9a3dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.apiusage; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -26,8 +27,7 @@ public interface TbApiUsageStateService { TenantApiUsageState getApiUsageState(TenantId tenantId); - void onAddedToAllowList(TenantId tenantId); - - void onAddedToDenyList(TenantId tenantId); + void onTenantProfileUpdate(TenantProfileId tenantProfileId); + void onTenantUpdate(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 29335c3326..0492ba9793 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -16,8 +16,13 @@ package org.thingsboard.server.service.apiusage; import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.tools.SchedulerUtils; import java.util.Map; @@ -29,7 +34,13 @@ public class TenantApiUsageState { private final Map currentHourValues = new ConcurrentHashMap<>(); @Getter - private final EntityId entityId; + @Setter + private TenantProfileId tenantProfileId; + @Getter + @Setter + private TenantProfileData tenantProfileData; + @Getter + private final ApiUsageState apiUsageState; @Getter private volatile long currentCycleTs; @Getter @@ -37,8 +48,10 @@ public class TenantApiUsageState { @Getter private volatile long currentHourTs; - public TenantApiUsageState(EntityId entityId) { - this.entityId = entityId; + public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) { + this.tenantProfileId = tenantProfile.getId(); + this.tenantProfileData = tenantProfile.getProfileData(); + this.apiUsageState = apiUsageState; this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); @@ -58,6 +71,10 @@ public class TenantApiUsageState { return result; } + public long get(ApiUsageRecordKey key) { + return currentCycleValues.getOrDefault(key, 0L); + } + public long addToHourly(ApiUsageRecordKey key, long value) { long result = currentHourValues.getOrDefault(key, 0L) + value; currentHourValues.put(key, result); @@ -79,4 +96,103 @@ public class TenantApiUsageState { } } + public long getProfileThreshold(ApiUsageRecordKey key) { + Object threshold = tenantProfileData.getProperties().get(key.name()); + if (threshold != null) { + if (threshold instanceof String) { + return Long.parseLong((String) threshold); + } else if (threshold instanceof Long) { + return (Long) threshold; + } + } + return 0L; + } + + public EntityId getEntityId() { + return apiUsageState.getEntityId(); + } + + public boolean isTransportEnabled() { + return apiUsageState.isTransportEnabled(); + } + + public boolean isDbStorageEnabled() { + return apiUsageState.isDbStorageEnabled(); + } + + public boolean isRuleEngineEnabled() { + return apiUsageState.isRuleEngineEnabled(); + } + + public boolean isJsExecEnabled() { + return apiUsageState.isJsExecEnabled(); + } + + public void setTransportEnabled(boolean transportEnabled) { + apiUsageState.setTransportEnabled(transportEnabled); + } + + public void setDbStorageEnabled(boolean dbStorageEnabled) { + apiUsageState.setDbStorageEnabled(dbStorageEnabled); + } + + public void setRuleEngineEnabled(boolean ruleEngineEnabled) { + apiUsageState.setRuleEngineEnabled(ruleEngineEnabled); + } + + public void setJsExecEnabled(boolean jsExecEnabled) { + apiUsageState.setJsExecEnabled(jsExecEnabled); + } + + public boolean isFeatureEnabled(ApiUsageRecordKey recordKey) { + switch (recordKey) { + case MSG_COUNT: + case MSG_BYTES_COUNT: + case DP_TRANSPORT_COUNT: + return isTransportEnabled(); + case RE_EXEC_COUNT: + return isRuleEngineEnabled(); + case DP_STORAGE_COUNT: + return isDbStorageEnabled(); + case JS_EXEC_COUNT: + return isJsExecEnabled(); + default: + return true; + } + } + + public boolean setFeatureValue(ApiUsageRecordKey recordKey, boolean value) { + boolean currentValue = isFeatureEnabled(recordKey); + switch (recordKey) { + case MSG_COUNT: + case MSG_BYTES_COUNT: + case DP_TRANSPORT_COUNT: + setTransportEnabled(value); + break; + case RE_EXEC_COUNT: + setRuleEngineEnabled(value); + break; + case DP_STORAGE_COUNT: + setDbStorageEnabled(value); + break; + case JS_EXEC_COUNT: + setJsExecEnabled(value); + break; + } + return currentValue == value; + } + + public boolean checkStateUpdatedDueToThresholds() { + boolean update = false; + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + update |= checkStateUpdatedDueToThreshold(key); + } + return update; + } + + public boolean checkStateUpdatedDueToThreshold(ApiUsageRecordKey recordKey) { + long value = get(recordKey); + long threshold = getProfileThreshold(recordKey); + return setFeatureValue(recordKey, threshold == 0 || value < threshold); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 929d5000f0..4ad073ead8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -107,7 +107,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> nfConsumer; public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, - TbDeviceProfileCache deviceProfileCache, TbQueueConsumer> nfConsumer) { + TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer> nfConsumer) { this.actorContext = actorContext; this.encodingService = encodingService; + this.tenantProfileCache = tenantProfileCache; this.deviceProfileCache = deviceProfileCache; + this.apiUsageStateService = apiUsageStateService; this.nfConsumer = nfConsumer; } @@ -143,7 +150,14 @@ public abstract class AbstractConsumerService implements HasTenan private static final long serialVersionUID = 8250339805336035966L; + @Getter @Setter private TenantId tenantId; + @Getter @Setter private EntityId entityId; + @Getter @Setter + private boolean transportEnabled; + @Getter @Setter + private boolean dbStorageEnabled; + @Getter @Setter + private boolean ruleEngineEnabled; + @Getter @Setter + private boolean jsExecEnabled; public ApiUsageState() { super(); @@ -43,22 +55,4 @@ public class ApiUsageState extends BaseData implements HasTenan this.tenantId = ur.getTenantId(); this.entityId = ur.getEntityId(); } - - @Override - public TenantId getTenantId() { - return tenantId; - } - - public void setTenantId(TenantId tenantId) { - this.tenantId = tenantId; - } - - public EntityId getEntityId() { - return entityId; - } - - public void setEntityId(EntityId entityId) { - this.entityId = entityId; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java index 6be56a1815..7dc390046e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java @@ -59,6 +59,14 @@ public class ApiApiUsageStateServiceImpl extends AbstractEntityService implement return apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); } + @Override + public ApiUsageState update(ApiUsageState apiUsageState) { + log.trace("Executing save [{}]", apiUsageState.getTenantId()); + validateId(apiUsageState.getTenantId(), INCORRECT_TENANT_ID + apiUsageState.getTenantId()); + validateId(apiUsageState.getId(), "Can't save new usage state. Only update is allowed!"); + return apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + } + @Override public ApiUsageState findTenantApiUsageState(TenantId tenantId) { log.trace("Executing findTenantUsageRecord, tenantId [{}]", tenantId); From 9ab92ba73cddfb584bebfd3166e41908ae989236 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 21 Oct 2020 10:21:01 +0300 Subject: [PATCH 12/30] validateApiUsageState improvements --- .../thingsboard/server/service/security/AccessValidator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java index 46de4f150e..57c192235e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java +++ b/application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java @@ -250,6 +250,9 @@ public class AccessValidator { if (currentUser.isSystemAdmin()) { callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); } else { + if (!operation.equals(Operation.READ_TELEMETRY)) { + callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!")); + } ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId())); if (apiUsageState == null) { callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!")); From a922b8227f299de8b7c0bb48a6cf322f2bc6a464 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 21 Oct 2020 10:35:15 +0300 Subject: [PATCH 13/30] Usage records compile fix --- .../server/service/queue/DefaultTbCoreConsumerService.java | 3 +++ .../service/queue/DefaultTbRuleEngineConsumerService.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 4ad073ead8..33c21a5b5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -54,6 +54,7 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; @@ -104,6 +105,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService Date: Wed, 21 Oct 2020 11:13:48 +0300 Subject: [PATCH 14/30] Cleanup of the map on repartition --- .../DefaultTbApiUsageStateService.java | 41 +++++++++++++++---- .../apiusage/TbApiUsageStateService.java | 6 ++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 84801224e1..003ee72807 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.tools.SchedulerUtils; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -33,6 +34,8 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; @@ -54,6 +57,7 @@ import java.util.concurrent.locks.ReentrantLock; public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "HOURLY_"; + private final PartitionService partitionService; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; private final SchedulerComponent scheduler; @@ -68,7 +72,8 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { private final Lock updateLock = new ReentrantLock(); - public DefaultTbApiUsageStateService(ApiUsageStateService apiUsageStateService, TimeseriesService tsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { + public DefaultTbApiUsageStateService(PartitionService partitionService, ApiUsageStateService apiUsageStateService, TimeseriesService tsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { + this.partitionService = partitionService; this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; this.scheduler = scheduler; @@ -120,8 +125,16 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { + tenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + } + } + @Override public TenantApiUsageState getApiUsageState(TenantId tenantId) { + //We should always get it from the map of from the database; return null; } @@ -132,11 +145,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { try { tenantStates.values().forEach(state -> { if (tenantProfile.getId().equals(state.getTenantProfileId())) { - state.setTenantProfileData(tenantProfile.getProfileData()); - if (state.checkStateUpdatedDueToThresholds()) { - apiUsageStateService.update(state.getApiUsageState()); - //TODO: send notification to cluster; - } + updateTenantState(state, tenantProfile); } }); } finally { @@ -146,7 +155,25 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { @Override public void onTenantUpdate(TenantId tenantId) { + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + updateLock.lock(); + try { + TenantApiUsageState state = tenantStates.get(tenantId); + if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) { + updateTenantState(state, tenantProfile); + } + } finally { + updateLock.unlock(); + } + } + + private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { + state.setTenantProfileData(tenantProfile.getProfileData()); + if (state.checkStateUpdatedDueToThresholds()) { + apiUsageStateService.update(state.getApiUsageState()); + //TODO: send notification to cluster; + } } private void checkStartOfNextCycle() { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index d17bd9a3dd..4816dc1037 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,13 +15,15 @@ */ package org.thingsboard.server.service.apiusage; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -public interface TbApiUsageStateService { +public interface TbApiUsageStateService extends ApplicationListener { void process(TbProtoQueueMsg msg, TbCallback callback); From 41212e11c01130eee0ca99127c5f12f49b60f9d4 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 22 Oct 2020 15:33:34 +0300 Subject: [PATCH 15/30] created tenant profile configuration --- .../DefaultTbApiUsageStateService.java | 2 +- .../apiusage/TbApiUsageStateService.java | 2 +- .../service/apiusage/TenantApiUsageState.java | 2 +- .../DefaultSystemDataLoaderService.java | 12 +- .../BaseTenantProfileControllerTest.java | 2 +- .../server/common/data/TenantProfile.java | 1 + .../server/common/data/TenantProfileType.java | 20 +++ .../DefaultTenantProfileConfiguration.java | 45 ++++++ .../profile/TenantProfileConfiguration.java | 36 +++++ .../profile}/TenantProfileData.java | 4 +- .../DefaultTransportRateLimitService.java | 2 +- .../dao/model/sql/TenantProfileEntity.java | 3 +- .../dao/tenant/TenantProfileServiceImpl.java | 4 +- .../service/BaseTenantProfileServiceTest.java | 11 +- .../home/components/home-components.module.ts | 4 + .../tenant-profile-autocomplete.component.ts | 1 - .../tenant-profile-data.component.html | 16 +- .../profile/tenant-profile-data.component.ts | 6 +- .../profile/tenant-profile.component.ts | 15 +- ...enant-profile-configuration.component.html | 141 ++++++++++++++++++ ...-tenant-profile-configuration.component.ts | 115 ++++++++++++++ ...enant-profile-configuration.component.html | 27 ++++ .../tenant-profile-configuration.component.ts | 103 +++++++++++++ ui-ngx/src/app/shared/models/tenant.model.ts | 53 ++++++- .../assets/locale/locale.constant-en_US.json | 33 +++- 25 files changed, 628 insertions(+), 32 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java rename common/data/src/main/java/org/thingsboard/server/common/data/{ => tenant/profile}/TenantProfileData.java (91%) create mode 100644 ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index df7e5013d2..2dd06c9957 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index 4816dc1037..dcd9d4823e 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 0492ba9793..9c39047742 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -20,7 +20,7 @@ import lombok.Setter; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.tools.SchedulerUtils; diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 1a0f9eca65..eeef99fff6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; @@ -127,6 +128,10 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { public void createDefaultTenantProfiles() throws Exception { tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfileData tenantProfileData = new TenantProfileData(); + DefaultTenantProfileConfiguration configuration = new DefaultTenantProfileConfiguration(); + tenantProfileData.setConfiguration(configuration); + TenantProfile isolatedTbCoreProfile = new TenantProfile(); isolatedTbCoreProfile.setDefault(false); isolatedTbCoreProfile.setName("Isolated TB Core"); @@ -134,6 +139,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); isolatedTbCoreProfile.setIsolatedTbCore(true); isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); + isolatedTbCoreProfile.setProfileData(tenantProfileData); try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); } catch (DataValidationException e) { @@ -147,6 +153,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); isolatedTbRuleEngineProfile.setIsolatedTbCore(false); isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); } catch (DataValidationException e) { @@ -160,6 +168,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); + isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData); + try { tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); } catch (DataValidationException e) { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index a4d615ae2b..311a93bf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -23,7 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 32f620f869..5e5f4a09de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -21,6 +21,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java new file mode 100644 index 0000000000..2b0ed502ac --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public enum TenantProfileType { + DEFAULT +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java new file mode 100644 index 0000000000..f361ec901c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.TenantProfileType; + +@Data +public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration { + + private long maxDevices; + private long maxAssets; + + private String transportTenantMsgRateLimit; + private String transportTenantTelemetryMsgRateLimit; + private String transportTenantTelemetryDataPointsRateLimit; + private String transportDeviceMsgRateLimit; + private String transportDeviceTelemetryMsgRateLimit; + private String transportDeviceTelemetryDataPointsRateLimit; + + private long maxTransportMessages; + private long maxTransportDataPoints; + private long maxREExecutions; + private long maxJSExecutions; + private long maxDPStorageDays; + private int maxRuleNodeExecutionsPerMessage; + + @Override + public TenantProfileType getType() { + return TenantProfileType.DEFAULT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java new file mode 100644 index 0000000000..07b184b389 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.tenant.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.TenantProfileType; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")}) +public interface TenantProfileConfiguration { + + @JsonIgnore + TenantProfileType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java similarity index 91% rename from common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index 54319276f1..6336cb4ace 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data; +package org.thingsboard.server.common.data.tenant.profile; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -26,6 +26,8 @@ import java.util.Map; @Data public class TenantProfileData { + private TenantProfileConfiguration configuration; + @JsonIgnore private Map properties = new HashMap<>(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java index 569f5446db..8e811a282f 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java @@ -18,9 +18,9 @@ package org.thingsboard.server.common.transport.limits; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java index 7c69f58935..7663682496 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java @@ -22,14 +22,13 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JsonBinaryType; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 939e2b7426..864679ce2a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -23,20 +23,18 @@ import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.util.mapping.JacksonUtil; import java.util.Arrays; import java.util.Collections; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index a9917126df..22830bed8c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -21,13 +21,12 @@ import org.junit.Test; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.util.mapping.JacksonUtil; import java.util.ArrayList; import java.util.Collections; @@ -188,8 +187,8 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { Assert.assertTrue(pageData.getData().isEmpty()); tenantProfiles.addAll(pageData.getData()); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile)); } @@ -224,8 +223,8 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { List tenantProfiles = new ArrayList<>(); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile)); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index d2e7cb0a28..ec7ccea660 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -113,6 +113,8 @@ import { AlarmScheduleInfoComponent } from './profile/alarm/alarm-schedule-info. import { AlarmScheduleDialogComponent } from '@home/components/profile/alarm/alarm-schedule-dialog.component'; import { EditAlarmDetailsDialogComponent } from './profile/alarm/edit-alarm-details-dialog.component'; import { AlarmRuleConditionDialogComponent } from '@home/components/profile/alarm/alarm-rule-condition-dialog.component'; +import { DefaultTenantProfileConfigurationComponent } from './profile/tenant/default-tenant-profile-configuration.component'; +import { TenantProfileConfigurationComponent } from './profile/tenant/tenant-profile-configuration.component'; @NgModule({ declarations: @@ -182,6 +184,8 @@ import { AlarmRuleConditionDialogComponent } from '@home/components/profile/alar FilterUserInfoDialogComponent, FilterPredicateValueComponent, TenantProfileAutocompleteComponent, + DefaultTenantProfileConfigurationComponent, + TenantProfileConfigurationComponent, TenantProfileDataComponent, TenantProfileComponent, TenantProfileDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts index 3cdb0b3a4e..9553e23528 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-autocomplete.component.ts @@ -204,7 +204,6 @@ export class TenantProfileAutocompleteComponent implements ControlValueAccessor, createTenantProfile($event: Event, profileName: string) { $event.preventDefault(); const tenantProfile: TenantProfile = { - id: null, name: profileName }; this.openTenantProfileDialog(tenantProfile, true); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html index a3d504d0e0..3d853940f4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.html @@ -16,9 +16,15 @@ -->
- - + + + +
tenant-profile.profile-configuration
+ + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts index 377b6f0978..cc6ce7a6ce 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile-data.component.ts @@ -62,7 +62,7 @@ export class TenantProfileDataComponent implements ControlValueAccessor, OnInit ngOnInit() { this.tenantProfileDataFormGroup = this.fb.group({ - tenantProfileData: [null, Validators.required] + configuration: [null, Validators.required] }); this.tenantProfileDataFormGroup.valueChanges.subscribe(() => { this.updateModel(); @@ -79,13 +79,13 @@ export class TenantProfileDataComponent implements ControlValueAccessor, OnInit } writeValue(value: TenantProfileData | null): void { - this.tenantProfileDataFormGroup.get('tenantProfileData').patchValue(value, {emitEvent: false}); + this.tenantProfileDataFormGroup.patchValue({configuration: value?.configuration}, {emitEvent: false}); } private updateModel() { let tenantProfileData: TenantProfileData = null; if (this.tenantProfileDataFormGroup.valid) { - tenantProfileData = this.tenantProfileDataFormGroup.getRawValue().tenantProfileData; + tenantProfileData = this.tenantProfileDataFormGroup.getRawValue(); } this.propagateChange(tenantProfileData); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 2aadb551de..7cca3ae414 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -18,7 +18,12 @@ import { Component, Inject, Input, Optional } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { TenantProfile } from '@app/shared/models/tenant.model'; +import { + createTenantProfileConfiguration, + TenantProfile, + TenantProfileData, + TenantProfileType +} from '@app/shared/models/tenant.model'; import { ActionNotificationShow } from '@app/core/notification/notification.actions'; import { TranslateService } from '@ngx-translate/core'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -56,7 +61,9 @@ export class TenantProfileComponent extends EntityComponent { name: [entity ? entity.name : '', [Validators.required]], isolatedTbCore: [entity ? entity.isolatedTbCore : false, []], isolatedTbRuleEngine: [entity ? entity.isolatedTbRuleEngine : false, []], - profileData: [entity && !this.isAdd ? entity.profileData : {}, []], + profileData: [entity && !this.isAdd ? entity.profileData : { + configuration: createTenantProfileConfiguration(TenantProfileType.DEFAULT) + } as TenantProfileData, []], description: [entity ? entity.description : '', []], } ); @@ -66,7 +73,9 @@ export class TenantProfileComponent extends EntityComponent { this.entityForm.patchValue({name: entity.name}); this.entityForm.patchValue({isolatedTbCore: entity.isolatedTbCore}); this.entityForm.patchValue({isolatedTbRuleEngine: entity.isolatedTbRuleEngine}); - this.entityForm.patchValue({profileData: entity.profileData}); + this.entityForm.patchValue({profileData: !this.isAdd ? entity.profileData: { + configuration: createTenantProfileConfiguration(TenantProfileType.DEFAULT) + } as TenantProfileData}); this.entityForm.patchValue({description: entity.description}); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html new file mode 100644 index 0000000000..cdb4febcc0 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -0,0 +1,141 @@ + +
+
+ + tenant-profile.maximum-devices + + + {{ 'tenant-profile.maximum-devices-required' | translate}} + + + {{ 'tenant-profile.maximum-devices-range' | translate}} + + + + tenant-profile.maximum-assets + + + {{ 'tenant-profile.maximum-assets-required' | translate}} + + + {{ 'tenant-profile.maximum-assets-range' | translate}} + + + + tenant-profile.max-transport-messages + + + {{ 'tenant-profile.max-transport-messages-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-required' | translate}} + + + + tenant-profile.max-transport-data-points + + + {{ 'tenant-profile.max-transport-data-points-required' | translate}} + + + {{ 'tenant-profile.max-transport-data-points-range' | translate}} + + + + tenant-profile.max-r-e-executions + + + {{ 'tenant-profile.max-r-e-executions-required' | translate}} + + + {{ 'tenant-profile.max-r-e-executions-range' | translate}} + + + + tenant-profile.max-j-s-executions + + + {{ 'tenant-profile.max-j-s-executions-required' | translate}} + + + {{ 'tenant-profile.max-j-s-executions-range' | translate}} + + + + tenant-profile.max-d-p-storage-days + + + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} + + + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} + + + + tenant-profile.max-rule-node-executions-per-message + + + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} + + + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} + + + + tenant-profile.transport-tenant-msg-rate-limit + + + + tenant-profile.transport-tenant-telemetry-msg-rate-limit + + + + tenant-profile.transport-tenant-telemetry-data-points-rate-limit + + + + tenant-profile.transport-device-msg-rate-limit + + + + tenant-profile.transport-device-telemetry-msg-rate-limit + + + + tenant-profile.transport-device-telemetry-data-points-rate-limit + + +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts new file mode 100644 index 0000000000..f83b59c35a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -0,0 +1,115 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { + DefaultTenantProfileConfiguration, + TenantProfileConfiguration, + TenantProfileType +} from '@shared/models/tenant.model'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-default-tenant-profile-configuration', + templateUrl: './default-tenant-profile-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DefaultTenantProfileConfigurationComponent), + multi: true + }] +}) +export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { + + defaultTenantProfileConfigurationFormGroup: FormGroup; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private fb: FormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.defaultTenantProfileConfigurationFormGroup = this.fb.group({ + configuration: this.fb.group({ + maxDevices: [null, [Validators.required, Validators.min(0)]], + maxAssets: [null, [Validators.required, Validators.min(0)]], + transportTenantMsgRateLimit: [null, []], + transportTenantTelemetryMsgRateLimit: [null, []], + transportTenantTelemetryDataPointsRateLimit: [null, []], + transportDeviceMsgRateLimit: [null, []], + transportDeviceTelemetryMsgRateLimit: [null, []], + transportDeviceTelemetryDataPointsRateLimit: [null, []], + maxTransportMessages: [null, [Validators.required, Validators.min(0)]], + maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], + maxREExecutions: [null, [Validators.required, Validators.min(0)]], + maxJSExecutions: [null, [Validators.required, Validators.min(0)]], + maxDPStorageDays: [null, [Validators.required, Validators.min(0)]], + maxRuleNodeExecutionsPerMessage: [null, [Validators.required, Validators.min(0)]] + }) + }); + this.defaultTenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.defaultTenantProfileConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.defaultTenantProfileConfigurationFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: DefaultTenantProfileConfiguration | null): void { + if (isDefinedAndNotNull(value)) { + this.defaultTenantProfileConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + } + } + + private updateModel() { + let configuration: TenantProfileConfiguration = null; + if (this.defaultTenantProfileConfigurationFormGroup.valid) { + configuration = this.defaultTenantProfileConfigurationFormGroup.getRawValue().configuration; + configuration.type = TenantProfileType.DEFAULT; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html new file mode 100644 index 0000000000..3204b7d2fb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.html @@ -0,0 +1,27 @@ + +
+
+ + + + +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts new file mode 100644 index 0000000000..4f5fa989d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/tenant-profile-configuration.component.ts @@ -0,0 +1,103 @@ +/// +/// Copyright © 2016-2020 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { deepClone } from '@core/utils'; +import { TenantProfileConfiguration, TenantProfileType } from '@shared/models/tenant.model'; + +@Component({ + selector: 'tb-tenant-profile-configuration', + templateUrl: './tenant-profile-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => TenantProfileConfigurationComponent), + multi: true + }] +}) +export class TenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { + + tenantProfileType = TenantProfileType; + + tenantProfileConfigurationFormGroup: FormGroup; + + private requiredValue: boolean; + get required(): boolean { + return this.requiredValue; + } + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + type: TenantProfileType; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private fb: FormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.tenantProfileConfigurationFormGroup = this.fb.group({ + configuration: [null, Validators.required] + }); + this.tenantProfileConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.tenantProfileConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.tenantProfileConfigurationFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: TenantProfileConfiguration | null): void { + this.type = value?.type; + const configuration = deepClone(value); + if (configuration) { + delete configuration.type; + } + this.tenantProfileConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + } + + private updateModel() { + let configuration: TenantProfileConfiguration = null; + if (this.tenantProfileConfigurationFormGroup.valid) { + configuration = this.tenantProfileConfigurationFormGroup.getRawValue().configuration; + configuration.type = this.type; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 378dca7c25..ad946efdb7 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -19,8 +19,59 @@ import { TenantId } from './id/tenant-id'; import { TenantProfileId } from '@shared/models/id/tenant-profile-id'; import { BaseData } from '@shared/models/base-data'; +export enum TenantProfileType { + DEFAULT = 'DEFAULT' +} + +export interface DefaultTenantProfileConfiguration { + maxDevices: number; + maxAssets: number; + + transportTenantMsgRateLimit?: string; + transportTenantTelemetryMsgRateLimit?: string; + transportTenantTelemetryDataPointsRateLimit?: string; + transportDeviceMsgRateLimit?: string; + transportDeviceTelemetryMsgRateLimit?: string; + transportDeviceTelemetryDataPointsRateLimit?: string; + + maxTransportMessages: number; + maxTransportDataPoints: number; + maxREExecutions: number; + maxJSExecutions: number; + maxDPStorageDays: number; + maxRuleNodeExecutionsPerMessage: number; +} + +export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; + +export interface TenantProfileConfiguration extends TenantProfileConfigurations { + type: TenantProfileType; +} + +export function createTenantProfileConfiguration(type: TenantProfileType): TenantProfileConfiguration { + let configuration: TenantProfileConfiguration = null; + if (type) { + switch (type) { + case TenantProfileType.DEFAULT: + const defaultConfiguration: DefaultTenantProfileConfiguration = { + maxDevices: 0, + maxAssets: 0, + maxTransportMessages: 0, + maxTransportDataPoints: 0, + maxREExecutions: 0, + maxJSExecutions: 0, + maxDPStorageDays: 0, + maxRuleNodeExecutionsPerMessage: 0 + }; + configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; + break; + } + } + return configuration; +} + export interface TenantProfileData { - [key: string]: string; + configuration: TenantProfileConfiguration; } export interface TenantProfile extends BaseData { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 082c2241b4..ca88616110 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1923,6 +1923,7 @@ "name": "Name", "name-required": "Name is required.", "data": "Profile data", + "profile-configuration": "Profile configuration", "description": "Description", "default": "Default", "delete-tenant-profile-title": "Are you sure you want to delete the tenant profile '{{tenantProfileName}}'?", @@ -1932,7 +1933,37 @@ "set-default-tenant-profile-title": "Are you sure you want to make the tenant profile '{{tenantProfileName}}' default?", "set-default-tenant-profile-text": "After the confirmation the tenant profile will be marked as default and will be used for new tenants with no profile specified.", "no-tenant-profiles-found": "No tenant profiles found.", - "create-new-tenant-profile": "Create a new one!" + "create-new-tenant-profile": "Create a new one!", + "maximum-devices": "Maximum number of devices (0 - unlimited)", + "maximum-devices-required": "Maximum number of devices is required.", + "maximum-devices-range": "Minimum number of devices can't be negative", + "maximum-assets": "Maximum number of assets (0 - unlimited)", + "maximum-assets-required": "Maximum number of assets is required.", + "maximum-assets-range": "Maximum number of assets can't be negative", + "transport-tenant-msg-rate-limit": "Transport tenant messages rate limit.", + "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages rate limit.", + "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points rate limit.", + "transport-device-msg-rate-limit": "Transport device messages rate limit.", + "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", + "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", + "max-transport-messages-required": "Maximum number of transport messages is required.", + "max-transport-messages-range": "Maximum number of transport messages can't be negative", + "max-transport-data-points": "Maximum number of transport data points (0 - unlimited)", + "max-transport-data-points-required": "Maximum number of transport data points is required.", + "max-transport-data-points-range": "Minimum number of transport data points can't be negative", + "max-r-e-executions": "Maximum number of Rule Engine executions (0 - unlimited)", + "max-r-e-executions-required": "Maximum number of Rule Engine executions is required.", + "max-r-e-executions-range": "Minimum number of Rule Engine executions can't be negative", + "max-j-s-executions": "Maximum number of JavaScript executions (0 - unlimited)", + "max-j-s-executions-required": "Maximum number of JavaScript executions is required.", + "max-j-s-executions-range": "Minimum number of JavaScript executions can't be negative", + "max-d-p-storage-days": "Maximum number of data points storage days (0 - unlimited)", + "max-d-p-storage-days-required": "Maximum number of data points storage days is required.", + "max-d-p-storage-days-range": "Minimum number of data points storage days can't be negative", + "max-rule-node-executions-per-message": "Maximum number of rule node executions per message (0 - unlimited)", + "max-rule-node-executions-per-message-required": "Maximum number of rule node executions per message is required.", + "max-rule-node-executions-per-message-range": "Minimum number of rule node executions per message can't be negative" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", From 0e0ab6ca3982d7e20c47267e1967010ad9e607f7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 22 Oct 2020 15:56:31 +0300 Subject: [PATCH 16/30] Improvements to API State --- .../server/service/apiusage/ApiFeature.java | 5 + .../DefaultTbApiUsageStateService.java | 92 ++++++++++++++----- .../apiusage/TbApiUsageStateService.java | 7 +- .../service/apiusage/TenantApiUsageState.java | 43 ++++++--- .../queue/DefaultTbClusterService.java | 27 ++++-- .../service/queue/TbClusterService.java | 3 + .../processing/AbstractConsumerService.java | 4 +- .../transport/DefaultTransportApiService.java | 22 +++-- .../server/common/data/ApiUsageRecordKey.java | 1 - .../server/common/data/ApiUsageState.java | 30 ++++-- .../MultipleTbQueueCallbackWrapper.java | 45 +++++++++ common/queue/src/main/proto/queue.proto | 1 + .../transport/mqtt/MqttTransportHandler.java | 9 +- .../common/transport/TransportService.java | 4 +- .../DefaultTransportRateLimitService.java | 11 ++- .../limits/TransportRateLimitService.java | 1 + .../limits/TransportRateLimitType.java | 3 +- .../service/DefaultTransportService.java | 17 ++-- .../DefaultTransportTenantProfileCache.java | 26 ++++-- .../server/dao/model/ModelConstants.java | 4 + .../dao/model/sql/ApiUsageStateEntity.java | 19 ++++ ...mpl.java => ApiUsageStateServiceImpl.java} | 4 +- .../resources/sql/schema-entities-hsql.sql | 4 + .../main/resources/sql/schema-entities.sql | 4 + .../service/BaseApiUsageStateServiceTest.java | 13 +++ 25 files changed, 310 insertions(+), 89 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java rename dao/src/main/java/org/thingsboard/server/dao/usagerecord/{ApiApiUsageStateServiceImpl.java => ApiUsageStateServiceImpl.java} (96%) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java new file mode 100644 index 0000000000..a893a55873 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java @@ -0,0 +1,5 @@ +package org.thingsboard.server.service.apiusage; + +public enum ApiFeature { + TRANSPORT, DB, RE, JS +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index df7e5013d2..c5e6cab16d 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.apiusage; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; @@ -33,15 +34,18 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; +import org.thingsboard.server.service.queue.TbClusterService; import javax.annotation.PostConstruct; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -57,12 +61,17 @@ import java.util.concurrent.locks.ReentrantLock; public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "HOURLY_"; + private final TbClusterService clusterService; private final PartitionService partitionService; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; private final SchedulerComponent scheduler; private final TbTenantProfileCache tenantProfileCache; - private final Map tenantStates = new ConcurrentHashMap<>(); + + // Tenants that should be processed on this server + private final Map myTenantStates = new ConcurrentHashMap<>(); + // Tenants that should be processed on other servers + private final Map otherTenantStates = new ConcurrentHashMap<>(); @Value("${usage.stats.report.enabled:true}") private boolean enabled; @@ -72,7 +81,13 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { private final Lock updateLock = new ReentrantLock(); - public DefaultTbApiUsageStateService(PartitionService partitionService, ApiUsageStateService apiUsageStateService, TimeseriesService tsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { + public DefaultTbApiUsageStateService(TbClusterService clusterService, + PartitionService partitionService, + ApiUsageStateService apiUsageStateService, + TimeseriesService tsService, + SchedulerComponent scheduler, + TbTenantProfileCache tenantProfileCache) { + this.clusterService = clusterService; this.partitionService = partitionService; this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; @@ -93,7 +108,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); TenantApiUsageState tenantState; List updatedEntries; - boolean stateUpdated = false; + Map result = new HashMap<>(); updateLock.lock(); try { tenantState = getOrFetchState(tenantId); @@ -110,32 +125,49 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(HOURLY + recordKey.name(), newHourlyValue))); - stateUpdated |= tenantState.checkStateUpdatedDueToThreshold(recordKey); + Pair update = tenantState.checkStateUpdatedDueToThreshold(recordKey); + if (update != null) { + result.put(update.getFirst(), update.getSecond()); + } } } finally { updateLock.unlock(); } tsService.save(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L); - if (stateUpdated) { - // Save new state into the database; - apiUsageStateService.update(tenantState.getApiUsageState()); - //TODO: clear cache on cluster repartition. - //TODO: update profiles on tenant and profile updates. - //TODO: broadcast to everyone notifications about enabled/disabled features. + if (!result.isEmpty()) { + persistAndNotify(tenantState, result); } } @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { - tenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); } } @Override - public TenantApiUsageState getApiUsageState(TenantId tenantId) { - //We should always get it from the map of from the database; - return null; + public ApiUsageState getApiUsageState(TenantId tenantId) { + if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { + TenantApiUsageState state = getOrFetchState(tenantId); + return state.getApiUsageState(); + } else { + ApiUsageState state = otherTenantStates.get(tenantId); + if (state == null) { + updateLock.lock(); + try { + state = otherTenantStates.get(tenantId); + if (state == null) { + state = apiUsageStateService.findTenantApiUsageState(tenantId); + otherTenantStates.put(tenantId, state); + } + } finally { + updateLock.unlock(); + } + } + return state; + } } @Override @@ -143,7 +175,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); updateLock.lock(); try { - tenantStates.values().forEach(state -> { + myTenantStates.values().forEach(state -> { if (tenantProfile.getId().equals(state.getTenantProfileId())) { updateTenantState(state, tenantProfile); } @@ -158,7 +190,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); updateLock.lock(); try { - TenantApiUsageState state = tenantStates.get(tenantId); + TenantApiUsageState state = myTenantStates.get(tenantId); if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) { updateTenantState(state, tenantProfile); } @@ -167,12 +199,28 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } + @Override + public void onApiUsageStateUpdate(TenantId tenantId) { + + } private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { state.setTenantProfileData(tenantProfile.getProfileData()); - if (state.checkStateUpdatedDueToThresholds()) { - apiUsageStateService.update(state.getApiUsageState()); - //TODO: send notification to cluster; + Map result = state.checkStateUpdatedDueToThresholds(); + if (!result.isEmpty()) { + persistAndNotify(state, result); + } + } + + private void persistAndNotify(TenantApiUsageState state, Map result) { + // TODO: + // 1. Broadcast to everyone notifications about enabled/disabled features. + // 2. Report rule engine and js executor metrics + // 4. UI for configuration of the thresholds + // 5. Max rule node executions per message. + apiUsageStateService.update(state.getApiUsageState()); + if (result.containsKey(ApiFeature.TRANSPORT)) { + clusterService.onApiStateChange(state.getApiUsageState(), null); } } @@ -180,7 +228,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { updateLock.lock(); try { long now = System.currentTimeMillis(); - tenantStates.values().forEach(state -> { + myTenantStates.values().forEach(state -> { if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) { state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth()); } @@ -191,7 +239,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } private TenantApiUsageState getOrFetchState(TenantId tenantId) { - TenantApiUsageState tenantState = tenantStates.get(tenantId); + TenantApiUsageState tenantState = myTenantStates.get(tenantId); if (tenantState == null) { ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); if (dbStateEntity == null) { @@ -223,7 +271,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } } - tenantStates.put(tenantId, tenantState); + myTenantStates.put(tenantId, tenantState); } catch (InterruptedException | ExecutionException e) { log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index 4816dc1037..e19c0a4154 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,7 @@ package org.thingsboard.server.service.apiusage; import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -27,9 +28,11 @@ public interface TbApiUsageStateService extends ApplicationListener msg, TbCallback callback); - TenantApiUsageState getApiUsageState(TenantId tenantId); + ApiUsageState getApiUsageState(TenantId tenantId); void onTenantProfileUpdate(TenantProfileId tenantProfileId); void onTenantUpdate(TenantId tenantId); + + void onApiUsageStateUpdate(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 0492ba9793..71bc3f1f6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +17,17 @@ package org.thingsboard.server.service.apiusage; import lombok.Getter; import lombok.Setter; +import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -103,11 +106,17 @@ public class TenantApiUsageState { return Long.parseLong((String) threshold); } else if (threshold instanceof Long) { return (Long) threshold; + } else if (threshold instanceof Integer) { + return (Integer) threshold; } } return 0L; } + public TenantId getTenantId() { + return apiUsageState.getTenantId(); + } + public EntityId getEntityId() { return apiUsageState.getEntityId(); } @@ -121,7 +130,7 @@ public class TenantApiUsageState { } public boolean isRuleEngineEnabled() { - return apiUsageState.isRuleEngineEnabled(); + return apiUsageState.isReExecEnabled(); } public boolean isJsExecEnabled() { @@ -137,7 +146,7 @@ public class TenantApiUsageState { } public void setRuleEngineEnabled(boolean ruleEngineEnabled) { - apiUsageState.setRuleEngineEnabled(ruleEngineEnabled); + apiUsageState.setReExecEnabled(ruleEngineEnabled); } public void setJsExecEnabled(boolean jsExecEnabled) { @@ -147,7 +156,6 @@ public class TenantApiUsageState { public boolean isFeatureEnabled(ApiUsageRecordKey recordKey) { switch (recordKey) { case MSG_COUNT: - case MSG_BYTES_COUNT: case DP_TRANSPORT_COUNT: return isTransportEnabled(); case RE_EXEC_COUNT: @@ -161,38 +169,47 @@ public class TenantApiUsageState { } } - public boolean setFeatureValue(ApiUsageRecordKey recordKey, boolean value) { + public ApiFeature setFeatureValue(ApiUsageRecordKey recordKey, boolean value) { + ApiFeature feature = null; boolean currentValue = isFeatureEnabled(recordKey); switch (recordKey) { case MSG_COUNT: - case MSG_BYTES_COUNT: case DP_TRANSPORT_COUNT: + feature = ApiFeature.TRANSPORT; setTransportEnabled(value); break; case RE_EXEC_COUNT: + feature = ApiFeature.RE; setRuleEngineEnabled(value); break; case DP_STORAGE_COUNT: + feature = ApiFeature.DB; setDbStorageEnabled(value); break; case JS_EXEC_COUNT: + feature = ApiFeature.JS; setJsExecEnabled(value); break; } - return currentValue == value; + return currentValue == value ? null : feature; } - public boolean checkStateUpdatedDueToThresholds() { - boolean update = false; + public Map checkStateUpdatedDueToThresholds() { + Map result = new HashMap<>(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { - update |= checkStateUpdatedDueToThreshold(key); + Pair featureUpdate = checkStateUpdatedDueToThreshold(key); + if (featureUpdate != null) { + result.put(featureUpdate.getFirst(), featureUpdate.getSecond()); + } } - return update; + return result; } - public boolean checkStateUpdatedDueToThreshold(ApiUsageRecordKey recordKey) { + public Pair checkStateUpdatedDueToThreshold(ApiUsageRecordKey recordKey) { long value = get(recordKey); long threshold = getProfileThreshold(recordKey); - return setFeatureValue(recordKey, threshold == 0 || value < threshold); + boolean featureValue = threshold == 0 || value < threshold; + ApiFeature feature = setFeatureValue(recordKey, featureValue); + return feature != null ? Pair.of(feature, featureValue) : null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 9451b58e9f..b26baf8d2e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; @@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -206,6 +208,12 @@ public class DefaultTbClusterService implements TbClusterService { onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback); } + @Override + public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) { + onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback); + broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED)); + } + @Override public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); @@ -221,13 +229,14 @@ public class DefaultTbClusterService implements TbClusterService { onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); } - public void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { - log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entity.getName()); + public void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { + String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName(); + log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName); TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() .setEntityType(entityid.getEntityType().name()) .setData(ByteString.copyFrom(encodingService.encode(entity))).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); - broadcast(transportMsg); + broadcast(transportMsg, callback); } private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { @@ -238,15 +247,16 @@ public class DefaultTbClusterService implements TbClusterService { .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); - broadcast(transportMsg); + broadcast(transportMsg, callback); } - private void broadcast(ToTransportMsg transportMsg) { + private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) { TbQueueProducer> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); Set tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); + TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null; for (String transportServiceId : tbTransportServices) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); - toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); + toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback); toTransportNfs.incrementAndGet(); } } @@ -256,7 +266,8 @@ public class DefaultTbClusterService implements TbClusterService { TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) - || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)) { + || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE) + || msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java index 838dbe0eef..b36af16716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.queue; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -63,4 +64,6 @@ public interface TbClusterService { void onTenantChange(Tenant tenant, TbQueueCallback callback); void onTenantDelete(Tenant tenant, TbQueueCallback callback); + + void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0532d2f9c8..02fe877cd0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -161,6 +161,8 @@ public abstract class AbstractConsumerService deviceCreationLocks = new ConcurrentHashMap<>(); public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, - TbTenantProfileCache tenantProfileCache, DeviceService deviceService, + TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService, RelationService relationService, DeviceCredentialsService deviceCredentialsService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, DeviceProvisionService deviceProvisionService) { this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; + this.apiUsageStateService = apiUsageStateService; this.deviceService = deviceService; this.relationService = relationService; this.deviceCredentialsService = deviceCredentialsService; @@ -316,18 +319,21 @@ public class DefaultTransportApiService implements TransportApiService { private ListenableFuture handle(GetEntityProfileRequestMsg requestMsg) { EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); - ByteString data; + GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder(); if (entityType.equals(EntityType.DEVICE_PROFILE)) { DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); - data = ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile))); } else if (entityType.equals(EntityType.TENANT)) { - TenantProfile tenantProfile = tenantProfileCache.get(new TenantId(entityUuid)); - data = ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)); + TenantId tenantId = new TenantId(entityUuid); + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId); + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile))); + builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state))); } else { throw new RuntimeException("Invalid entity profile request: " + entityType); } - return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(GetEntityProfileResponseMsg.newBuilder().setData(data).build()).build()); + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build()); } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java index 9c8690a88f..bc730e7e5a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -18,7 +18,6 @@ package org.thingsboard.server.common.data; public enum ApiUsageRecordKey { MSG_COUNT, - MSG_BYTES_COUNT, DP_TRANSPORT_COUNT, DP_STORAGE_COUNT, RE_EXEC_COUNT, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java index 05186c7599..5144861790 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -29,18 +29,24 @@ public class ApiUsageState extends BaseData implements HasTenan private static final long serialVersionUID = 8250339805336035966L; - @Getter @Setter + @Getter + @Setter private TenantId tenantId; - @Getter @Setter + @Getter + @Setter private EntityId entityId; - @Getter @Setter - private boolean transportEnabled; - @Getter @Setter - private boolean dbStorageEnabled; - @Getter @Setter - private boolean ruleEngineEnabled; - @Getter @Setter - private boolean jsExecEnabled; + @Getter + @Setter + private boolean transportEnabled = true; + @Getter + @Setter + private boolean dbStorageEnabled = true; + @Getter + @Setter + private boolean reExecEnabled = true; + @Getter + @Setter + private boolean jsExecEnabled = true; public ApiUsageState() { super(); @@ -54,5 +60,9 @@ public class ApiUsageState extends BaseData implements HasTenan super(ur); this.tenantId = ur.getTenantId(); this.entityId = ur.getEntityId(); + this.transportEnabled = ur.isTransportEnabled(); + this.dbStorageEnabled = ur.isDbStorageEnabled(); + this.reExecEnabled = ur.isReExecEnabled(); + this.jsExecEnabled = ur.isJsExecEnabled(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java new file mode 100644 index 0000000000..ddedb3a606 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbQueueCallbackWrapper implements TbQueueCallback { + + private final AtomicInteger tbQueueCallbackCount; + private final TbQueueCallback callback; + + public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) { + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (tbQueueCallbackCount.decrementAndGet() <= 0) { + callback.onSuccess(metadata); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(new RuleEngineException(t.getMessage())); + } +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index f716bd90ae..4b5a6028e6 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -186,6 +186,7 @@ message GetEntityProfileRequestMsg { message GetEntityProfileResponseMsg { string entityType = 1; bytes data = 2; + bytes apiState = 3; } message EntityUpdateMsg { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 2c4b2db7df..e70da7029e 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; @@ -630,7 +631,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void onError(Throwable e) { - log.warn("[{}] Failed to submit session event", sessionId, e); + if (e instanceof TbRateLimitsException) { + log.trace("[{}] Failed to submit session event", sessionId, e); + } else { + log.warn("[{}] Failed to submit session event", sessionId, e); + } ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.close(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 4872d8e477..3b775b9f26 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -45,7 +45,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { - GetEntityProfileResponseMsg getRoutingInfo(GetEntityProfileRequestMsg msg); + GetEntityProfileResponseMsg getEntityProfile(GetEntityProfileRequestMsg msg); void process(DeviceTransportType transportType, ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); @@ -62,8 +62,6 @@ public interface TransportService { void process(ProvisionDeviceRequestMsg msg, TransportServiceCallback callback); - void onProfileUpdate(DeviceProfile deviceProfile); - boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints, TransportRateLimitType... limits); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java index 569f5446db..9663ae2e47 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentMap; @Slf4j public class DefaultTransportRateLimitService implements TransportRateLimitService { + private final ConcurrentMap tenantAllowed = new ConcurrentHashMap<>(); private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); @@ -46,6 +47,9 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi @Override public TransportRateLimitType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints, TransportRateLimitType... limits) { + if (!tenantAllowed.getOrDefault(tenantId, Boolean.TRUE)) { + return TransportRateLimitType.TENANT_ADDED_TO_DISABLED_LIST; + } TransportRateLimit[] tenantLimits = getTenantRateLimits(tenantId); TransportRateLimit[] deviceLimits = getDeviceRateLimits(tenantId, deviceId); for (TransportRateLimitType limitType : limits) { @@ -85,6 +89,11 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi perDeviceLimits.remove(deviceId); } + @Override + public void update(TenantId tenantId, boolean allowed) { + tenantAllowed.put(tenantId, allowed); + } + private void mergeLimits(TenantId tenantId, TransportRateLimit[] newRateLimits) { TransportRateLimit[] oldRateLimits = perTenantLimits.get(tenantId); if (oldRateLimits == null) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java index a97fbfc61d..2ab1ec38ac 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java @@ -31,4 +31,5 @@ public interface TransportRateLimitService { void remove(DeviceId deviceId); + void update(TenantId tenantId, boolean transportEnabled); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java index a3e6da6683..038de86e84 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ import lombok.Getter; public enum TransportRateLimitType { + TENANT_ADDED_TO_DISABLED_LIST("general.tenant.disabled", true, false), TENANT_MAX_MSGS("transport.tenant.msg", true, true), TENANT_TELEMETRY_MSGS("transport.tenant.telemetry", true, true), TENANT_MAX_DATA_POINTS("transport.tenant.dataPoints", true, false), diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 010a3cd899..326f996d14 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,6 +26,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; @@ -237,7 +238,7 @@ public class DefaultTransportService implements TransportService { } @Override - public TransportProtos.GetEntityProfileResponseMsg getRoutingInfo(TransportProtos.GetEntityProfileRequestMsg msg) { + public TransportProtos.GetEntityProfileResponseMsg getEntityProfile(TransportProtos.GetEntityProfileRequestMsg msg) { TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setEntityProfileRequestMsg(msg).build()); try { @@ -641,8 +642,7 @@ public class DefaultTransportService implements TransportService { onProfileUpdate(deviceProfile); } } else if (EntityType.TENANT_PROFILE.equals(entityType)) { - TenantProfileUpdateResult update = tenantProfileCache.put(msg.getData()); - rateLimitService.update(update); + rateLimitService.update(tenantProfileCache.put(msg.getData())); } else if (EntityType.TENANT.equals(entityType)) { Optional profileOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); if (profileOpt.isPresent()) { @@ -652,6 +652,12 @@ public class DefaultTransportService implements TransportService { rateLimitService.update(tenant.getId()); } } + } else if (EntityType.API_USAGE_STATE.equals(entityType)) { + Optional stateOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); + if (stateOpt.isPresent()) { + ApiUsageState apiUsageState = stateOpt.get(); + rateLimitService.update(apiUsageState.getTenantId(), apiUsageState.isTransportEnabled()); + } } } else if (toSessionMsg.hasEntityDeleteMsg()) { TransportProtos.EntityDeleteMsg msg = toSessionMsg.getEntityDeleteMsg(); @@ -673,8 +679,7 @@ public class DefaultTransportService implements TransportService { } } - @Override - public void onProfileUpdate(DeviceProfile deviceProfile) { + private void onProfileUpdate(DeviceProfile deviceProfile) { long deviceProfileIdMSB = deviceProfile.getId().getId().getMostSignificantBits(); long deviceProfileIdLSB = deviceProfile.getId().getId().getLeastSignificantBits(); sessions.forEach((id, md) -> { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 784f7b3ae8..18c1512a27 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,21 +18,19 @@ package org.thingsboard.server.common.transport.service; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; +import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.util.TbTransportComponent; import java.util.Collections; @@ -54,8 +52,15 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil private final ConcurrentMap> tenantProfileIds = new ConcurrentHashMap<>(); private final DataDecodingEncodingService dataDecodingEncodingService; + private TransportRateLimitService rateLimitService; private TransportService transportService; + @Lazy + @Autowired + public void setRateLimitService(TransportRateLimitService rateLimitService) { + this.rateLimitService = rateLimitService; + } + @Lazy @Autowired public void setTransportService(TransportService transportService) { @@ -77,7 +82,8 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil if (profileOpt.isPresent()) { TenantProfile newProfile = profileOpt.get(); log.trace("[{}] put: {}", newProfile.getId(), newProfile); - return new TenantProfileUpdateResult(newProfile, tenantProfileIds.get(newProfile.getId())); + Set affectedTenants = tenantProfileIds.get(newProfile.getId()); + return new TenantProfileUpdateResult(newProfile, affectedTenants != null ? affectedTenants : Collections.emptySet()); } else { log.warn("Failed to decode profile: {}", profileBody.toString()); return new TenantProfileUpdateResult(null, Collections.emptySet()); @@ -127,8 +133,8 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil .setEntityIdMSB(tenantId.getId().getMostSignificantBits()) .setEntityIdLSB(tenantId.getId().getLeastSignificantBits()) .build(); - TransportProtos.GetEntityProfileResponseMsg routingInfo = transportService.getRoutingInfo(msg); - Optional profileOpt = dataDecodingEncodingService.decode(routingInfo.getData().toByteArray()); + TransportProtos.GetEntityProfileResponseMsg entityProfileMsg = transportService.getEntityProfile(msg); + Optional profileOpt = dataDecodingEncodingService.decode(entityProfileMsg.getData().toByteArray()); if (profileOpt.isPresent()) { profile = profileOpt.get(); TenantProfile existingProfile = profiles.get(profile.getId()); @@ -140,9 +146,11 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil tenantProfileIds.computeIfAbsent(profile.getId(), id -> ConcurrentHashMap.newKeySet()).add(tenantId); tenantIds.put(tenantId, profile.getId()); } else { - log.warn("[{}] Can't decode tenant profile: {}", tenantId, routingInfo.getData()); + log.warn("[{}] Can't decode tenant profile: {}", tenantId, entityProfileMsg.getData()); throw new RuntimeException("Can't decode tenant profile!"); } + Optional apiStateOpt = dataDecodingEncodingService.decode(entityProfileMsg.getApiState().toByteArray()); + apiStateOpt.ifPresent(apiUsageState -> rateLimitService.update(tenantId, apiUsageState.isTransportEnabled())); } } finally { tenantProfileFetchLock.unlock(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index a0f5b45f02..a370fbe7f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -445,6 +445,10 @@ public class ModelConstants { public static final String API_USAGE_STATE_TENANT_ID_COLUMN = TENANT_ID_PROPERTY; public static final String API_USAGE_STATE_ENTITY_TYPE_COLUMN = ENTITY_TYPE_COLUMN; public static final String API_USAGE_STATE_ENTITY_ID_COLUMN = ENTITY_ID_COLUMN; + public static final String API_USAGE_STATE_TRANSPORT_ENABLED_COLUMN = "transport_enabled"; + public static final String API_USAGE_STATE_DB_STORAGE_ENABLED_COLUMN = "db_storage_enabled"; + public static final String API_USAGE_STATE_RE_EXEC_ENABLED_COLUMN = "re_exec_enabled"; + public static final String API_USAGE_STATE_JS_EXEC_ENABLED_COLUMN = "js_exec_enabled"; /** * Cassandra attributes and timeseries constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java index a06ec142d3..262252517a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -51,6 +53,15 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements @Column(name = ModelConstants.API_USAGE_STATE_ENTITY_ID_COLUMN) private UUID entityId; + @Column(name = ModelConstants.API_USAGE_STATE_TRANSPORT_ENABLED_COLUMN) + private boolean transportEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_DB_STORAGE_ENABLED_COLUMN) + private boolean dbStorageEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_RE_EXEC_ENABLED_COLUMN) + private boolean reExecEnabled = true; + @Column(name = ModelConstants.API_USAGE_STATE_JS_EXEC_ENABLED_COLUMN) + private boolean jsExecEnabled = true; + public ApiUsageStateEntity() { } @@ -66,6 +77,10 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements this.entityType = ur.getEntityId().getEntityType().name(); this.entityId = ur.getEntityId().getId(); } + this.transportEnabled = ur.isTransportEnabled(); + this.dbStorageEnabled = ur.isDbStorageEnabled(); + this.reExecEnabled = ur.isReExecEnabled(); + this.jsExecEnabled = ur.isJsExecEnabled(); } @Override @@ -78,6 +93,10 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements if (entityId != null) { ur.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } + ur.setTransportEnabled(transportEnabled); + ur.setDbStorageEnabled(dbStorageEnabled); + ur.setReExecEnabled(reExecEnabled); + ur.setJsExecEnabled(jsExecEnabled); return ur; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java rename to dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index e59c990346..15be73a0b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -31,13 +31,13 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -public class ApiApiUsageStateServiceImpl extends AbstractEntityService implements ApiUsageStateService { +public class ApiUsageStateServiceImpl extends AbstractEntityService implements ApiUsageStateService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private final ApiUsageStateDao apiUsageStateDao; private final TenantDao tenantDao; - public ApiApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao) { + public ApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao) { this.tenantDao = tenantDao; this.apiUsageStateDao = apiUsageStateDao; } diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index c671cd2e19..1d82e2d493 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -411,5 +411,9 @@ CREATE TABLE IF NOT EXISTS api_usage_state ( tenant_id uuid, entity_type varchar(32), entity_id uuid, + transport_enabled boolean, + db_storage_enabled boolean, + re_exec_enabled boolean, + js_exec_enabled boolean, CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index d8539d9af9..c26fce6080 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -437,6 +437,10 @@ CREATE TABLE IF NOT EXISTS api_usage_state ( tenant_id uuid, entity_type varchar(32), entity_id uuid, + transport_enabled boolean, + db_storage_enabled boolean, + re_exec_enabled boolean, + js_exec_enabled boolean, CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java index 5f56506997..a33e190a03 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseApiUsageStateServiceTest.java @@ -48,4 +48,17 @@ public abstract class BaseApiUsageStateServiceTest extends AbstractServiceTest { Assert.assertNotNull(apiUsageState); } + @Test + public void testUpdateApiUsageState(){ + ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); + Assert.assertTrue(apiUsageState.isTransportEnabled()); + apiUsageState.setTransportEnabled(false); + apiUsageState = apiUsageStateService.update(apiUsageState); + Assert.assertNotNull(apiUsageState); + apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(apiUsageState); + Assert.assertFalse(apiUsageState.isTransportEnabled()); + } + } From b38b47f4093b547502c3e47a72f7f88526e45032 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 22 Oct 2020 18:51:34 +0300 Subject: [PATCH 17/30] JS Invoke Service stats --- .../actors/ruleChain/DefaultTbContext.java | 2 +- .../controller/RuleChainController.java | 2 +- .../server/service/apiusage/ApiFeature.java | 15 ++ .../service/apiusage/TenantApiUsageState.java | 36 +-- .../queue/DefaultTbClusterService.java | 2 +- .../processing/AbstractConsumerService.java | 11 +- .../script/AbstractJsInvokeService.java | 13 +- .../AbstractNashornJsInvokeService.java | 11 +- .../service/script/JsExecutorService.java | 2 +- .../service/script/JsInvokeService.java | 5 +- .../script/NashornJsInvokeService.java | 5 + .../service/script/RemoteJsInvokeService.java | 5 + .../script/RuleNodeJsScriptEngine.java | 11 +- .../transport/DefaultTransportApiService.java | 2 +- .../script/RuleNodeJsScriptEngineTest.java | 250 ------------------ .../script/TestNashornJsInvokeService.java | 57 ---- .../server/common/data/ApiUsageRecordKey.java | 6 +- .../tenant/profile/TenantProfileData.java | 13 - .../MultipleTbQueueCallbackWrapper.java | 2 +- .../transport/mqtt/MqttTransportHandler.java | 2 +- .../common/transport/TransportService.java | 5 - .../DefaultTransportRateLimitFactory.java | 47 ---- .../DefaultTransportRateLimitService.java | 128 +++++---- ...ry.java => EntityTransportRateLimits.java} | 11 +- .../limits/SimpleTransportRateLimit.java | 5 + .../limits/TransportRateLimitService.java | 3 +- .../limits/TransportRateLimitType.java | 48 ---- .../service/DefaultTransportService.java | 32 +-- .../DefaultTransportTenantProfileCache.java | 2 +- .../dao/tenant/TenantProfileServiceImpl.java | 5 +- .../service/BaseTenantProfileServiceTest.java | 5 +- 31 files changed, 209 insertions(+), 534 deletions(-) delete mode 100644 application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java rename common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/{TransportRateLimitFactory.java => EntityTransportRateLimits.java} (70%) delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index c8bb083577..e7a54385a7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -312,7 +312,7 @@ class DefaultTbContext implements TbContext { @Override public ScriptEngine createJsScriptEngine(String script, String... argNames) { - return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); + return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 6d80c05c05..2ca834d85b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -347,7 +347,7 @@ public class RuleChainController extends BaseController { String errorText = ""; ScriptEngine engine = null; try { - engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); + engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames); TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java index a893a55873..8267d4addd 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.apiusage; public enum ApiFeature { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 0e68d02b71..5dcfb34ec3 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,7 @@ import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -100,15 +101,18 @@ public class TenantApiUsageState { } public long getProfileThreshold(ApiUsageRecordKey key) { - Object threshold = tenantProfileData.getProperties().get(key.name()); - if (threshold != null) { - if (threshold instanceof String) { - return Long.parseLong((String) threshold); - } else if (threshold instanceof Long) { - return (Long) threshold; - } else if (threshold instanceof Integer) { - return (Integer) threshold; - } + DefaultTenantProfileConfiguration config = (DefaultTenantProfileConfiguration) tenantProfileData.getConfiguration(); + switch (key) { + case TRANSPORT_MSG_COUNT: + return config.getMaxTransportMessages(); + case TRANSPORT_DP_COUNT: + return config.getMaxTransportDataPoints(); + case JS_EXEC_COUNT: + return config.getMaxJSExecutions(); + case RE_EXEC_COUNT: + return config.getMaxREExecutions(); + case STORAGE_DP_COUNT: + return config.getMaxDPStorageDays(); } return 0L; } @@ -155,12 +159,12 @@ public class TenantApiUsageState { public boolean isFeatureEnabled(ApiUsageRecordKey recordKey) { switch (recordKey) { - case MSG_COUNT: - case DP_TRANSPORT_COUNT: + case TRANSPORT_MSG_COUNT: + case TRANSPORT_DP_COUNT: return isTransportEnabled(); case RE_EXEC_COUNT: return isRuleEngineEnabled(); - case DP_STORAGE_COUNT: + case STORAGE_DP_COUNT: return isDbStorageEnabled(); case JS_EXEC_COUNT: return isJsExecEnabled(); @@ -173,8 +177,8 @@ public class TenantApiUsageState { ApiFeature feature = null; boolean currentValue = isFeatureEnabled(recordKey); switch (recordKey) { - case MSG_COUNT: - case DP_TRANSPORT_COUNT: + case TRANSPORT_MSG_COUNT: + case TRANSPORT_DP_COUNT: feature = ApiFeature.TRANSPORT; setTransportEnabled(value); break; @@ -182,7 +186,7 @@ public class TenantApiUsageState { feature = ApiFeature.RE; setRuleEngineEnabled(value); break; - case DP_STORAGE_COUNT: + case STORAGE_DP_COUNT: feature = ApiFeature.DB; setDbStorageEnabled(value); break; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b26baf8d2e..f08092d32d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 02fe877cd0..9a4d488e0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -153,10 +154,14 @@ public abstract class AbstractConsumerService scriptIdToNameMap = new ConcurrentHashMap<>(); protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected AbstractJsInvokeService(TbUsageStatsClient apiUsageStatsClient) { + this.apiUsageStatsClient = apiUsageStatsClient; + } + public void init(long maxRequestsTimeout) { if (maxRequestsTimeout > 0) { timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nashorn-js-timeout")); @@ -50,7 +58,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } @Override - public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { + public ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) { UUID scriptId = UUID.randomUUID(); String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); @@ -58,12 +66,13 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } @Override - public ListenableFuture invokeFunction(UUID scriptId, Object... args) { + public ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args) { String functionName = scriptIdToNameMap.get(scriptId); if (functionName == null) { return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); } if (!isBlackListed(scriptId)) { + apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); return doInvokeFunction(scriptId, functionName, args); } else { return Futures.immediateFailedFuture( diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 8e6666bbba..17be0602f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,7 +38,6 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); - @Autowired @Getter - private JsExecutorService jsExecutor; + private final JsExecutorService jsExecutor; @Value("${js.local.max_requests_timeout:0}") private long maxRequestsTimeout; @@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @Value("${js.local.stats.enabled:false}") private boolean statsEnabled; + public AbstractNashornJsInvokeService(TbUsageStatsClient apiUsageStatsClient, JsExecutorService jsExecutor) { + super(apiUsageStatsClient); + this.jsExecutor = jsExecutor; + } + @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java index 97e40e6f62..15fd8c7534 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java @@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor { @Override protected int getThreadPollSize() { - return jsExecutorThreadPoolSize; + return Math.max(jsExecutorThreadPoolSize, 1); } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java index a9f727e4a7..ae990af66c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java @@ -17,14 +17,15 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; public interface JsInvokeService { - ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames); + ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames); - ListenableFuture invokeFunction(UUID scriptId, Object... args); + ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args); ListenableFuture release(UUID scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 66a14cc827..c20f3e8c08 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; import java.util.concurrent.TimeUnit; @@ -42,6 +43,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; + public NashornJsInvokeService(TbUsageStatsClient apiUsageStatsClient, JsExecutorService jsExecutor) { + super(apiUsageStatsClient, jsExecutor); + } + @Override protected boolean useJsSandbox() { return useJsSandbox; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index cfb0c8a8ee..2ada577287 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -68,6 +69,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); + public RemoteJsInvokeService(TbUsageStatsClient apiUsageStatsClient) { + super(apiUsageStatsClient); + } + @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 110f410fad..275e645ce2 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private final JsInvokeService sandboxService; private final UUID scriptId; + private final TenantId tenantId; private final EntityId entityId; - public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { + this.tenantId = tenantId; this.sandboxService = sandboxService; this.entityId = entityId; try { - this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); + this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); } catch (Exception e) { Throwable t = e; if (e instanceof ExecutionException) { @@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private JsonNode executeScript(TbMsg msg) throws ScriptException { try { String[] inArgs = prepareArgs(msg); - String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); + String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); return mapper.readTree(eval); } catch (ExecutionException e) { if (e.getCause() instanceof ScriptException) { @@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S private ListenableFuture executeScriptAsync(TbMsg msg) { String[] inArgs = prepareArgs(msg); - return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]), o -> { try { return Futures.immediateFuture(mapper.readTree(o.toString())); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6d0b7a0aa8..ab566fb147 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java deleted file mode 100644 index 60495b119d..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.collect.Sets; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.rule.engine.api.ScriptEngine; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; - -import javax.script.ScriptException; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.*; - -public class RuleNodeJsScriptEngineTest { - - private ScriptEngine scriptEngine; - private TestNashornJsInvokeService jsSandboxService; - - private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased()); - - @Before - public void beforeTest() throws Exception { - jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3); - } - - @After - public void afterTest() throws Exception { - jsSandboxService.stop(); - } - - @Test - public void msgCanBeUpdated() throws ScriptException { - String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("70", actual.getMetaData().getValue("temp")); - scriptEngine.destroy(); - } - - @Test - public void newAttributesCanBeAddedInMsg() throws ScriptException { - String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - assertEquals("94", actual.getMetaData().getValue("newAttr")); - scriptEngine.destroy(); - } - - @Test - public void payloadCanBeUpdated() throws ScriptException { - String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - - TbMsg actual = scriptEngine.executeUpdate(msg); - - String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}"; - assertEquals(expectedJson, actual.getData()); - scriptEngine.destroy(); - } - - @Test - public void metadataAccessibleForFilter() throws ScriptException { - String function = "return metadata.humidity < 15;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); - assertFalse(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForFilter() throws ScriptException { - String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "7"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); - assertTrue(scriptEngine.executeFilter(msg)); - scriptEngine.destroy(); - } - - @Test - public void dataAccessibleForSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return 'one'\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one"), actual); - scriptEngine.destroy(); - } - - @Test - public void multipleRelationsReturnedFromSwitch() throws ScriptException { - String jsCode = "function nextRelation(metadata, msg) {\n" + - " if(msg.passed == 5 && metadata.temp == 10)\n" + - " return ['three', 'one']\n" + - " else\n" + - " return 'two';\n" + - "};\n" + - "\n" + - "return nextRelation(metadata, msg);"; - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("temp", "10"); - metaData.putValue("humidity", "99"); - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); - Set actual = scriptEngine.executeSwitch(msg); - assertEquals(Sets.newHashSet("one", "three"), actual); - scriptEngine.destroy(); - } - - @Test - public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException { - String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};"; - - int repeat = 1000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - } - - finishLatch.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - - CountDownLatch nextStart = new CountDownLatch(repeat); - CountDownLatch nextFinish = new CountDownLatch(repeat); - for (int i = 0; i < repeat; i++) { - service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code)); - } - - nextFinish.await(); - assertTrue(scriptIds.size() == 1); - assertTrue(failedCount.get() == 0); - service.shutdownNow(); - } - - @Test - public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException { - String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};"; - - int repeat = 10000; - ExecutorService service = Executors.newFixedThreadPool(repeat); - Map scriptIds = new ConcurrentHashMap<>(); - CountDownLatch startLatch = new CountDownLatch(repeat); - CountDownLatch finishLatch = new CountDownLatch(repeat); - AtomicInteger failedCount = new AtomicInteger(0); - for (int i = 0; i < repeat; i++) { - service.submit(() -> { - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code)); - }); - } - - finishLatch.await(); - assertTrue(scriptIds.isEmpty()); - assertEquals(repeat, failedCount.get()); - service.shutdownNow(); - } - - private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount, - Map scriptIds, String code) { - try { - for (int k = 0; k < 10; k++) { - startLatch.countDown(); - startLatch.await(); - UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get(); - scriptIds.put(scriptId, new Object()); - jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get(); - jsSandboxService.release(scriptId).get(); - } - } catch (Throwable th) { - failedCount.incrementAndGet(); - } finally { - finishLatch.countDown(); - } - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java deleted file mode 100644 index 8a89de5dff..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.script; - -public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { - - private boolean useJsSandbox; - private final int monitorThreadPoolSize; - private final long maxCpuTime; - private final int maxErrors; - - public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) { - this.useJsSandbox = useJsSandbox; - this.monitorThreadPoolSize = monitorThreadPoolSize; - this.maxCpuTime = maxCpuTime; - this.maxErrors = maxErrors; - init(); - } - - @Override - protected boolean useJsSandbox() { - return useJsSandbox; - } - - @Override - protected int getMonitorThreadPoolSize() { - return monitorThreadPoolSize; - } - - @Override - protected long getMaxCpuTime() { - return maxCpuTime; - } - - @Override - protected int getMaxErrors() { - return maxErrors; - } - - @Override - protected long getMaxBlacklistDuration() { - return 100000; - } -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java index bc730e7e5a..0048e325f3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -17,9 +17,9 @@ package org.thingsboard.server.common.data; public enum ApiUsageRecordKey { - MSG_COUNT, - DP_TRANSPORT_COUNT, - DP_STORAGE_COUNT, + TRANSPORT_MSG_COUNT, + TRANSPORT_DP_COUNT, + STORAGE_DP_COUNT, RE_EXEC_COUNT, JS_EXEC_COUNT diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index 6336cb4ace..0996fc9850 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -28,17 +28,4 @@ public class TenantProfileData { private TenantProfileConfiguration configuration; - @JsonIgnore - private Map properties = new HashMap<>(); - - @JsonAnyGetter - public Map properties() { - return this.properties; - } - - @JsonAnySetter - public void put(String name, Object value) { - this.properties.put(name, value); - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java index ddedb3a606..c28f3637b7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index e70da7029e..14712a7cb3 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 3b775b9f26..980aa33221 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -19,7 +19,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.limits.TransportRateLimitType; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; @@ -62,10 +61,6 @@ public interface TransportService { void process(ProvisionDeviceRequestMsg msg, TransportServiceCallback callback); - boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); - - boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints, TransportRateLimitType... limits); - void process(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java deleted file mode 100644 index 362502bf17..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.limits; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.msg.tools.TbRateLimits; - -@Slf4j -@Component -public class DefaultTransportRateLimitFactory implements TransportRateLimitFactory { - - private static final DummyTransportRateLimit ALWAYS_TRUE = new DummyTransportRateLimit(); - - @Override - public TransportRateLimit create(TransportRateLimitType type, Object configuration) { - if (!StringUtils.isEmpty(configuration)) { - try { - return new SimpleTransportRateLimit(new TbRateLimits(configuration.toString()), configuration.toString()); - } catch (Exception e) { - log.warn("[{}] Failed to init rate limit with configuration: {}", type, configuration, e); - return ALWAYS_TRUE; - } - } else { - return ALWAYS_TRUE; - } - } - - @Override - public TransportRateLimit createDefault(TransportRateLimitType type) { - return ALWAYS_TRUE; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java index bc415cd92a..b6f160ad1e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,76 +17,96 @@ package org.thingsboard.server.common.transport.limits; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.queue.util.TbTransportComponent; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; +import java.util.function.Function; @Service @TbTransportComponent @Slf4j public class DefaultTransportRateLimitService implements TransportRateLimitService { + private final static DummyTransportRateLimit ALLOW = new DummyTransportRateLimit(); private final ConcurrentMap tenantAllowed = new ConcurrentHashMap<>(); - private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); - private final TransportRateLimitFactory rateLimitFactory; private final TransportTenantProfileCache tenantProfileCache; - public DefaultTransportRateLimitService(TransportRateLimitFactory rateLimitFactory, TransportTenantProfileCache tenantProfileCache) { - this.rateLimitFactory = rateLimitFactory; + public DefaultTransportRateLimitService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @Override - public TransportRateLimitType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints, TransportRateLimitType... limits) { + public EntityType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints) { if (!tenantAllowed.getOrDefault(tenantId, Boolean.TRUE)) { - return TransportRateLimitType.TENANT_ADDED_TO_DISABLED_LIST; + return EntityType.TENANT; } - TransportRateLimit[] tenantLimits = getTenantRateLimits(tenantId); - TransportRateLimit[] deviceLimits = getDeviceRateLimits(tenantId, deviceId); - for (TransportRateLimitType limitType : limits) { - TransportRateLimit rateLimit; - if (limitType.isTenantLevel()) { - rateLimit = tenantLimits[limitType.ordinal()]; - } else { - rateLimit = deviceLimits[limitType.ordinal()]; - } - if (!rateLimit.tryConsume(limitType.isMessageLevel() ? 1L : dataPoints)) { - return limitType; - } + if (!checkEntityRateLimit(dataPoints, getTenantRateLimits(tenantId))) { + return EntityType.TENANT; + } + if (!checkEntityRateLimit(dataPoints, getDeviceRateLimits(tenantId, deviceId))) { + return EntityType.DEVICE; } return null; } + private boolean checkEntityRateLimit(int dataPoints, EntityTransportRateLimits tenantLimits) { + if (dataPoints > 0) { + return tenantLimits.getTelemetryMsgRateLimit().tryConsume() && tenantLimits.getTelemetryDataPointsRateLimit().tryConsume(dataPoints); + } else { + return tenantLimits.getRegularMsgRateLimit().tryConsume(); + } + } + @Override public void update(TenantProfileUpdateResult update) { - TransportRateLimit[] newLimits = createTransportRateLimits(update.getProfile()); + EntityTransportRateLimits tenantRateLimitPrototype = createRateLimits(update.getProfile(), true); + EntityTransportRateLimits deviceRateLimitPrototype = createRateLimits(update.getProfile(), false); for (TenantId tenantId : update.getAffectedTenants()) { - mergeLimits(tenantId, newLimits); + mergeLimits(tenantId, tenantRateLimitPrototype, perTenantLimits::get, perTenantLimits::put); + tenantDevices.get(tenantId).forEach(deviceId -> { + mergeLimits(deviceId, deviceRateLimitPrototype, perDeviceLimits::get, perDeviceLimits::put); + }); } } @Override public void update(TenantId tenantId) { - mergeLimits(tenantId, fetchProfileAndInit(tenantId)); + EntityTransportRateLimits tenantRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), true); + EntityTransportRateLimits deviceRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), false); + mergeLimits(tenantId, tenantRateLimitPrototype, perTenantLimits::get, perTenantLimits::put); + tenantDevices.get(tenantId).forEach(deviceId -> { + mergeLimits(deviceId, deviceRateLimitPrototype, perDeviceLimits::get, perDeviceLimits::put); + }); } @Override public void remove(TenantId tenantId) { perTenantLimits.remove(tenantId); + tenantDevices.remove(tenantId); } @Override public void remove(DeviceId deviceId) { perDeviceLimits.remove(deviceId); + tenantDevices.values().forEach(set -> set.remove(deviceId)); } @Override @@ -94,48 +114,66 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi tenantAllowed.put(tenantId, allowed); } - private void mergeLimits(TenantId tenantId, TransportRateLimit[] newRateLimits) { - TransportRateLimit[] oldRateLimits = perTenantLimits.get(tenantId); + private void mergeLimits(T deviceId, EntityTransportRateLimits newRateLimits, + Function getFunction, + BiConsumer putFunction) { + EntityTransportRateLimits oldRateLimits = getFunction.apply(deviceId); if (oldRateLimits == null) { - perTenantLimits.put(tenantId, newRateLimits); + putFunction.accept(deviceId, newRateLimits); } else { - for (int i = 0; i < TransportRateLimitType.values().length; i++) { - TransportRateLimit newLimit = newRateLimits[i]; - TransportRateLimit oldLimit = oldRateLimits[i]; - if (newLimit != null && (oldLimit == null || !oldLimit.getConfiguration().equals(newLimit.getConfiguration()))) { - oldRateLimits[i] = newLimit; - } + EntityTransportRateLimits updated = merge(oldRateLimits, newRateLimits); + if (updated != null) { + putFunction.accept(deviceId, updated); } } } - private TransportRateLimit[] fetchProfileAndInit(TenantId tenantId) { - return perTenantLimits.computeIfAbsent(tenantId, tmp -> createTransportRateLimits(tenantProfileCache.get(tenantId))); + private EntityTransportRateLimits merge(EntityTransportRateLimits oldRateLimits, EntityTransportRateLimits newRateLimits) { + boolean regularUpdate = !oldRateLimits.getRegularMsgRateLimit().getConfiguration().equals(newRateLimits.getRegularMsgRateLimit().getConfiguration()); + boolean telemetryMsgRateUpdate = !oldRateLimits.getTelemetryMsgRateLimit().getConfiguration().equals(newRateLimits.getTelemetryMsgRateLimit().getConfiguration()); + boolean telemetryDataPointUpdate = !oldRateLimits.getTelemetryDataPointsRateLimit().getConfiguration().equals(newRateLimits.getTelemetryDataPointsRateLimit().getConfiguration()); + if (regularUpdate || telemetryMsgRateUpdate || telemetryDataPointUpdate) { + return new EntityTransportRateLimits( + regularUpdate ? newLimit(newRateLimits.getRegularMsgRateLimit().getConfiguration()) : oldRateLimits.getRegularMsgRateLimit(), + telemetryMsgRateUpdate ? newLimit(newRateLimits.getTelemetryMsgRateLimit().getConfiguration()) : oldRateLimits.getTelemetryMsgRateLimit(), + telemetryDataPointUpdate ? newLimit(newRateLimits.getTelemetryDataPointsRateLimit().getConfiguration()) : oldRateLimits.getTelemetryDataPointsRateLimit()); + } else { + return null; + } } - private TransportRateLimit[] createTransportRateLimits(TenantProfile tenantProfile) { + private EntityTransportRateLimits createRateLimits(TenantProfile tenantProfile, boolean tenant) { TenantProfileData profileData = tenantProfile.getProfileData(); - TransportRateLimit[] rateLimits = new TransportRateLimit[TransportRateLimitType.values().length]; - for (TransportRateLimitType type : TransportRateLimitType.values()) { - rateLimits[type.ordinal()] = rateLimitFactory.create(type, profileData.getProperties().get(type.getConfigurationKey())); + DefaultTenantProfileConfiguration profile = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); + if (profile == null) { + return new EntityTransportRateLimits(ALLOW, ALLOW, ALLOW); + } else { + TransportRateLimit regularMsgRateLimit = newLimit(tenant ? profile.getTransportTenantMsgRateLimit() : profile.getTransportDeviceMsgRateLimit()); + TransportRateLimit telemetryMsgRateLimit = newLimit(tenant ? profile.getTransportTenantTelemetryMsgRateLimit() : profile.getTransportDeviceTelemetryMsgRateLimit()); + TransportRateLimit telemetryDpRateLimit = newLimit(tenant ? profile.getTransportTenantTelemetryDataPointsRateLimit() : profile.getTransportTenantTelemetryDataPointsRateLimit()); + return new EntityTransportRateLimits(regularMsgRateLimit, telemetryMsgRateLimit, telemetryDpRateLimit); } - return rateLimits; } - private TransportRateLimit[] getTenantRateLimits(TenantId tenantId) { - TransportRateLimit[] limits = perTenantLimits.get(tenantId); + private static TransportRateLimit newLimit(String config) { + return StringUtils.isEmpty(config) ? ALLOW : new SimpleTransportRateLimit(config); + } + + private EntityTransportRateLimits getTenantRateLimits(TenantId tenantId) { + EntityTransportRateLimits limits = perTenantLimits.get(tenantId); if (limits == null) { - limits = fetchProfileAndInit(tenantId); + limits = createRateLimits(tenantProfileCache.get(tenantId), true); perTenantLimits.put(tenantId, limits); } return limits; } - private TransportRateLimit[] getDeviceRateLimits(TenantId tenantId, DeviceId deviceId) { - TransportRateLimit[] limits = perDeviceLimits.get(deviceId); + private EntityTransportRateLimits getDeviceRateLimits(TenantId tenantId, DeviceId deviceId) { + EntityTransportRateLimits limits = perDeviceLimits.get(deviceId); if (limits == null) { - limits = fetchProfileAndInit(tenantId); + limits = createRateLimits(tenantProfileCache.get(tenantId), false); perDeviceLimits.put(deviceId, limits); + tenantDevices.computeIfAbsent(tenantId, id -> ConcurrentHashMap.newKeySet()).add(deviceId); } return limits; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java similarity index 70% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java index 54d83610c2..50831ccff7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/EntityTransportRateLimits.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.common.transport.limits; -public interface TransportRateLimitFactory { +import lombok.AllArgsConstructor; +import lombok.Data; - TransportRateLimit create(TransportRateLimitType type, Object config); +@Data +@AllArgsConstructor +public class EntityTransportRateLimits { - TransportRateLimit createDefault(TransportRateLimitType type); + private TransportRateLimit regularMsgRateLimit; + private TransportRateLimit telemetryMsgRateLimit; + private TransportRateLimit telemetryDataPointsRateLimit; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java index 3253272ded..0ea920a4ba 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/SimpleTransportRateLimit.java @@ -26,6 +26,11 @@ public class SimpleTransportRateLimit implements TransportRateLimit { @Getter private final String configuration; + public SimpleTransportRateLimit(String configuration) { + this.configuration = configuration; + this.rateLimit = new TbRateLimits(configuration); + } + @Override public boolean tryConsume() { return rateLimit.tryConsume(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java index 2ab1ec38ac..3918927c0b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitService.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.common.transport.limits; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; public interface TransportRateLimitService { - TransportRateLimitType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints, TransportRateLimitType... limits); + EntityType checkLimits(TenantId tenantId, DeviceId deviceId, int dataPoints); void update(TenantProfileUpdateResult update); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java deleted file mode 100644 index 038de86e84..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.limits; - -import lombok.Getter; - -public enum TransportRateLimitType { - - TENANT_ADDED_TO_DISABLED_LIST("general.tenant.disabled", true, false), - TENANT_MAX_MSGS("transport.tenant.msg", true, true), - TENANT_TELEMETRY_MSGS("transport.tenant.telemetry", true, true), - TENANT_MAX_DATA_POINTS("transport.tenant.dataPoints", true, false), - DEVICE_MAX_MSGS("transport.device.msg", false, true), - DEVICE_TELEMETRY_MSGS("transport.device.telemetry", false, true), - DEVICE_MAX_DATA_POINTS("transport.device.dataPoints", false, false); - - @Getter - private final String configurationKey; - @Getter - private final boolean tenantLevel; - @Getter - private final boolean deviceLevel; - @Getter - private final boolean messageLevel; - @Getter - private final boolean dataPointLevel; - - TransportRateLimitType(String configurationKey, boolean tenantLevel, boolean messageLevel) { - this.configurationKey = configurationKey; - this.tenantLevel = tenantLevel; - this.deviceLevel = !tenantLevel; - this.messageLevel = messageLevel; - this.dataPointLevel = !messageLevel; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 326f996d14..f77a3ce44b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -55,8 +55,6 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.limits.TransportRateLimitType; -import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; @@ -364,7 +362,7 @@ public class DefaultTransportService implements TransportService { for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { dataPoints += tsKv.getKvCount(); } - if (checkLimits(sessionInfo, msg, callback, dataPoints, TELEMETRY)) { + if (checkLimits(sessionInfo, msg, callback, dataPoints)) { reportActivityInternal(sessionInfo); TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); @@ -385,7 +383,7 @@ public class DefaultTransportService implements TransportService { @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { - if (checkLimits(sessionInfo, msg, callback, msg.getKvCount(), TELEMETRY)) { + if (checkLimits(sessionInfo, msg, callback, msg.getKvCount())) { reportActivityInternal(sessionInfo); TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); @@ -575,31 +573,23 @@ public class DefaultTransportService implements TransportService { sessions.remove(toSessionId(sessionInfo)); } - private TransportRateLimitType[] DEFAULT = new TransportRateLimitType[]{TransportRateLimitType.TENANT_MAX_MSGS, TransportRateLimitType.DEVICE_MAX_MSGS}; - private TransportRateLimitType[] TELEMETRY = TransportRateLimitType.values(); - - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { - return checkLimits(sessionInfo, msg, callback, 0, DEFAULT); + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + return checkLimits(sessionInfo, msg, callback, 0); } - @Override - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints, TransportRateLimitType... limits) { + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints) { if (log.isTraceEnabled()) { log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); - TransportRateLimitType limit = rateLimitService.checkLimits(tenantId, deviceId, 0, limits); - if (limit == null) { + EntityType rateLimitedEntityType = rateLimitService.checkLimits(tenantId, deviceId, dataPoints); + if (rateLimitedEntityType == null) { return true; } else { if (callback != null) { - callback.onError(new TbRateLimitsException(limit.isTenantLevel() ? EntityType.TENANT : EntityType.DEVICE)); - } - if (log.isTraceEnabled()) { - log.trace("[{}][{}] {} rateLimit detected: {}", toSessionId(sessionInfo), tenantId, limit, msg); + callback.onError(new TbRateLimitsException(rateLimitedEntityType)); } return false; } @@ -831,8 +821,8 @@ public class DefaultTransportService implements TransportService { @Override public void onSuccess(T msg) { try { - apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.MSG_COUNT, 1); - apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.DP_TRANSPORT_COUNT, dataPoints); + apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_MSG_COUNT, 1); + apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_DP_COUNT, dataPoints); } finally { callback.onSuccess(msg); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 18c1512a27..6773d09417 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 864679ce2a..af327d3505 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -145,7 +146,9 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T defaultTenantProfile = new TenantProfile(); defaultTenantProfile.setDefault(true); defaultTenantProfile.setName("Default"); - defaultTenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData profileData = new TenantProfileData(); + profileData.setConfiguration(new DefaultTenantProfileConfiguration()); + defaultTenantProfile.setProfileData(profileData); defaultTenantProfile.setDescription("Default tenant profile"); defaultTenantProfile.setIsolatedTbCore(false); defaultTenantProfile.setIsolatedTbRuleEngine(false); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index 22830bed8c..f93fc2fb1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.exception.DataValidationException; @@ -262,7 +263,9 @@ public class BaseTenantProfileServiceTest extends AbstractServiceTest { TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName(name); tenantProfile.setDescription(name + " Test"); - tenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData profileData = new TenantProfileData(); + profileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(profileData); tenantProfile.setDefault(false); tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbRuleEngine(false); From 03581c294161fdbebe38b775c080e8f9dad5a4d1 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 27 Oct 2020 18:30:23 +0200 Subject: [PATCH 18/30] Usage records --- .../server/actors/ActorSystemContext.java | 11 +- .../RuleChainActorMessageProcessor.java | 8 +- .../ruleChain/RuleChainManagerActor.java | 6 + .../RuleNodeActorMessageProcessor.java | 5 + .../server/actors/tenant/TenantActor.java | 63 ++++++---- .../server/service/apiusage/ApiFeature.java | 11 +- .../DefaultTbApiUsageStateService.java | 110 ++++++++++++------ .../service/apiusage/TenantApiUsageState.java | 24 +--- .../DefaultSystemDataLoaderService.java | 3 +- .../script/AbstractJsInvokeService.java | 61 ++++++---- .../AbstractNashornJsInvokeService.java | 8 +- .../script/NashornJsInvokeService.java | 7 +- .../service/script/RemoteJsInvokeService.java | 7 +- .../server/common/data/ApiUsageRecordKey.java | 22 +++- .../DefaultTenantProfileConfiguration.java | 19 +++ .../profile/TenantProfileConfiguration.java | 4 + ...ient.java => DefaultTbApiUsageClient.java} | 8 +- ...StatsClient.java => TbApiUsageClient.java} | 3 +- .../service/DefaultTransportService.java | 13 ++- .../sql/tenant/TenantProfileRepository.java | 1 + .../server/dao/tenant/TenantProfileDao.java | 1 + .../usagerecord/ApiUsageStateServiceImpl.java | 35 +++++- 22 files changed, 295 insertions(+), 135 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/queue/usagestats/{DefaultTbUsageStatsClient.java => DefaultTbApiUsageClient.java} (93%) rename common/queue/src/main/java/org/thingsboard/server/queue/usagestats/{TbUsageStatsClient.java => TbApiUsageClient.java} (90%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7f028932f2..5a0e291ac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -106,6 +107,14 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired + @Getter + private TbApiUsageStateService apiUsageStateService; + + @Autowired + @Getter + private TbApiUsageClient apiUsageClient; + @Autowired @Getter @Setter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 5ca394be35..f5df62cfd3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.queue.TbClusterService; import java.util.ArrayList; @@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor> nodeRoutes; private final RuleChainService service; private final TbClusterService clusterService; + private final TbApiUsageClient apiUsageClient; private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext - , TbActorRef parent, TbActorRef self) { + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) { super(systemContext, tenantId, ruleChain.getId()); + this.apiUsageClient = systemContext.getApiUsageClient(); this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; @@ -331,6 +334,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + ctx.stop(new TbEntityActorId(ruleChain.getId())); + } + } + protected void visit(RuleChain entity, TbActorRef actorRef) { if (entity != null && entity.isRoot()) { rootChain = entity; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index db55ff8edf..52bd9f8c28 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -21,6 +21,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.shared.ComponentMsgProcessor; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; /** * @author Andrew Shvayka @@ -36,6 +38,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngineForCurrentTenant) { try { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { - log.info("[{}] Going to init rule chains", tenantId); - initRuleChains(); + if (apiUsageState.isReExecEnabled()) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } } else { isRuleEngineForCurrentTenant = false; } @@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor { } } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); -// TODO: throw this in 3.1? -// throw new TbActorException("Failed to init actor", e); } } @@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor { if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; queueMsg.getTbMsg().getCallback().onSuccess(); - } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ + } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) { TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; transportMsg.getCallback().onSuccess(); } @@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor { return; } TbMsg tbMsg = msg.getTbMsg(); - if (tbMsg.getRuleChainId() == null) { - if (getRootChainActor() != null) { - getRootChainActor().tell(msg); + if (apiUsageState.isReExecEnabled()) { + if (tbMsg.getRuleChainId() == null) { + if (getRootChainActor() != null) { + getRootChainActor().tell(msg); + } else { + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); + log.info("[{}] No Root Chain: {}", tenantId, msg); + } } else { - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); - log.info("[{}] No Root Chain: {}", tenantId, msg); + try { + ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); + } catch (TbActorNotRegisteredException ex) { + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); + //TODO: 3.1 Log it to dead letters queue; + tbMsg.getCallback().onSuccess(); + } } } else { - try { - ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); - } catch (TbActorNotRegisteredException ex) { - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId()); - //TODO: 3.1 Log it to dead letters queue; - tbMsg.getCallback().onSuccess(); - } + log.trace("[{}] Ack message because Rule Engine is disabled", tenantId); + tbMsg.getCallback().onSuccess(); } } private void onRuleChainMsg(RuleChainAwareMsg msg) { - getOrCreateActor(msg.getRuleChainId()).tell(msg); + if (apiUsageState.isReExecEnabled()) { + getOrCreateActor(msg.getRuleChainId()).tell(msg); + } } private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { @@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor { } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { + if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) { + ApiUsageState old = apiUsageState; + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId)); + if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId); + destroyRuleChains(); + } else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) { + log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId); + initRuleChains(); + } + } if (isRuleEngineForCurrentTenant) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java index 8267d4addd..f2dfab54d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java @@ -15,6 +15,15 @@ */ package org.thingsboard.server.service.apiusage; +import lombok.Getter; + public enum ApiFeature { - TRANSPORT, DB, RE, JS + TRANSPORT("transportApiState"), DB("dbApiState"), RE("ruleEngineApiState"), JS("jsExecutionApiState"); + + @Getter + private final String apiStateKey; + + ApiFeature(String apiStateKey) { + this.apiStateKey = apiStateKey; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 61d0baca0c..1b1a2853c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -15,18 +15,24 @@ */ package org.thingsboard.server.service.apiusage; +import com.google.common.util.concurrent.FutureCallback; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.tools.SchedulerUtils; @@ -34,7 +40,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; -import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; @@ -42,6 +47,7 @@ import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.PostConstruct; import java.util.ArrayList; @@ -60,11 +66,23 @@ import java.util.concurrent.locks.ReentrantLock; @Service public class DefaultTbApiUsageStateService implements TbApiUsageStateService { - public static final String HOURLY = "HOURLY_"; + public static final String HOURLY = "Hourly"; + public static final FutureCallback VOID_CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; private final TbClusterService clusterService; private final PartitionService partitionService; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; + private final TelemetrySubscriptionService tsWsService; private final SchedulerComponent scheduler; private final TbTenantProfileCache tenantProfileCache; @@ -84,13 +102,14 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public DefaultTbApiUsageStateService(TbClusterService clusterService, PartitionService partitionService, ApiUsageStateService apiUsageStateService, - TimeseriesService tsService, + TimeseriesService tsService, TelemetrySubscriptionService tsWsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { this.clusterService = clusterService; this.partitionService = partitionService; this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; + this.tsWsService = tsWsService; this.scheduler = scheduler; this.tenantProfileCache = tenantProfileCache; } @@ -122,9 +141,9 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); long newValue = tenantState.add(recordKey, kvProto.getValue()); - updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.name(), newValue))); + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue))); long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); - updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(HOURLY + recordKey.name(), newHourlyValue))); + updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue))); Pair update = tenantState.checkStateUpdatedDueToThreshold(recordKey); if (update != null) { result.put(update.getFirst(), update.getSecond()); @@ -133,10 +152,11 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } finally { updateLock.unlock(); } - tsService.save(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L); + tsWsService.saveAndNotify(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L, VOID_CALLBACK); if (!result.isEmpty()) { persistAndNotify(tenantState, result); } + callback.onSuccess(); } @Override @@ -149,27 +169,38 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { @Override public ApiUsageState getApiUsageState(TenantId tenantId) { - if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { - TenantApiUsageState state = getOrFetchState(tenantId); - return state.getApiUsageState(); + TenantApiUsageState tenantState = myTenantStates.get(tenantId); + if (tenantState != null) { + return tenantState.getApiUsageState(); } else { ApiUsageState state = otherTenantStates.get(tenantId); - if (state == null) { - updateLock.lock(); - try { - state = otherTenantStates.get(tenantId); - if (state == null) { - state = apiUsageStateService.findTenantApiUsageState(tenantId); - otherTenantStates.put(tenantId, state); + if (state != null) { + return state; + } else { + if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { + return getOrFetchState(tenantId).getApiUsageState(); + } else { + updateLock.lock(); + try { + state = otherTenantStates.get(tenantId); + if (state == null) { + state = apiUsageStateService.findTenantApiUsageState(tenantId); + otherTenantStates.put(tenantId, state); + } + } finally { + updateLock.unlock(); } - } finally { - updateLock.unlock(); + return state; } } - return state; } } + @Override + public void onApiUsageStateUpdate(TenantId tenantId) { + otherTenantStates.remove(tenantId); + } + @Override public void onTenantProfileUpdate(TenantProfileId tenantProfileId) { TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); @@ -199,29 +230,38 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } - @Override - public void onApiUsageStateUpdate(TenantId tenantId) { - - } - private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { + TenantProfileData oldProfileData = state.getTenantProfileData(); state.setTenantProfileData(tenantProfile.getProfileData()); Map result = state.checkStateUpdatedDueToThresholds(); if (!result.isEmpty()) { persistAndNotify(state, result); } + updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(), + oldProfileData.getConfiguration(), tenantProfile.getProfileData().getConfiguration()); + } + + private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id, + TenantProfileConfiguration oldData, TenantProfileConfiguration newData) { + long ts = System.currentTimeMillis(); + List profileThresholds = new ArrayList<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + if (oldData == null || oldData.getProfileThreshold(key) != newData.getProfileThreshold(key)) { + profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newData.getProfileThreshold(key)))); + } + } + if (!profileThresholds.isEmpty()) { + tsWsService.saveAndNotify(tenantId, id, profileThresholds, 0L, VOID_CALLBACK); + } } private void persistAndNotify(TenantApiUsageState state, Map result) { - // TODO: - // 1. Broadcast to everyone notifications about enabled/disabled features. - // 2. Report rule engine and js executor metrics - // 4. UI for configuration of the thresholds - // 5. Max rule node executions per message. apiUsageStateService.update(state.getApiUsageState()); - if (result.containsKey(ApiFeature.TRANSPORT)) { - clusterService.onApiStateChange(state.getApiUsageState(), null); - } + clusterService.onApiStateChange(state.getApiUsageState(), null); + long ts = System.currentTimeMillis(); + List stateTelemetry = new ArrayList<>(); + result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState))))); + tsWsService.saveAndNotify(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, 0L, VOID_CALLBACK); } private void checkStartOfNextCycle() { @@ -252,15 +292,15 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity); try { - List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getEntityId()).get(); + List dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { boolean cycleEntryFound = false; boolean hourlyEntryFound = false; for (TsKvEntry tsKvEntry : dbValues) { - if (tsKvEntry.getKey().equals(key.name())) { + if (tsKvEntry.getKey().equals(key.getApiCountKey())) { cycleEntryFound = true; tenantState.put(key, tsKvEntry.getLongValue().get()); - } else if (tsKvEntry.getKey().equals(HOURLY + key.name())) { + } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { hourlyEntryFound = true; if (tsKvEntry.getTs() == tenantState.getCurrentHourTs()) { tenantState.putHourly(key, tsKvEntry.getLongValue().get()); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java index 5dcfb34ec3..8fcc1d46cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java @@ -21,11 +21,10 @@ import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.tools.SchedulerUtils; import java.util.HashMap; @@ -101,30 +100,13 @@ public class TenantApiUsageState { } public long getProfileThreshold(ApiUsageRecordKey key) { - DefaultTenantProfileConfiguration config = (DefaultTenantProfileConfiguration) tenantProfileData.getConfiguration(); - switch (key) { - case TRANSPORT_MSG_COUNT: - return config.getMaxTransportMessages(); - case TRANSPORT_DP_COUNT: - return config.getMaxTransportDataPoints(); - case JS_EXEC_COUNT: - return config.getMaxJSExecutions(); - case RE_EXEC_COUNT: - return config.getMaxREExecutions(); - case STORAGE_DP_COUNT: - return config.getMaxDPStorageDays(); - } - return 0L; + return tenantProfileData.getConfiguration().getProfileThreshold(key); } public TenantId getTenantId() { return apiUsageState.getTenantId(); } - public EntityId getEntityId() { - return apiUsageState.getEntityId(); - } - public boolean isTransportEnabled() { return apiUsageState.isTransportEnabled(); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index eeef99fff6..e6d44f9de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -129,8 +129,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); TenantProfileData tenantProfileData = new TenantProfileData(); - DefaultTenantProfileConfiguration configuration = new DefaultTenantProfileConfiguration(); - tenantProfileData.setConfiguration(configuration); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); TenantProfile isolatedTbCoreProfile = new TenantProfile(); isolatedTbCoreProfile.setDefault(false); diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index eb5d3e3b49..8d9fa898dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -21,7 +21,8 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import java.util.Map; import java.util.UUID; @@ -36,13 +37,15 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public abstract class AbstractJsInvokeService implements JsInvokeService { - private final TbUsageStatsClient apiUsageStatsClient; + private final TbApiUsageStateService apiUsageStateService; + private final TbApiUsageClient apiUsageClient; protected ScheduledExecutorService timeoutExecutorService; protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map disabledFunctions = new ConcurrentHashMap<>(); - protected AbstractJsInvokeService(TbUsageStatsClient apiUsageStatsClient) { - this.apiUsageStatsClient = apiUsageStatsClient; + protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + this.apiUsageStateService = apiUsageStateService; + this.apiUsageClient = apiUsageClient; } public void init(long maxRequestsTimeout) { @@ -59,24 +62,32 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @Override public ListenableFuture eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) { - UUID scriptId = UUID.randomUUID(); - String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); - String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); - return doEval(scriptId, functionName, jsScript); + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + UUID scriptId = UUID.randomUUID(); + String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); + String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); + return doEval(scriptId, functionName, jsScript); + } else { + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); + } } @Override public ListenableFuture invokeFunction(TenantId tenantId, UUID scriptId, Object... args) { - String functionName = scriptIdToNameMap.get(scriptId); - if (functionName == null) { - return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); - } - if (!isBlackListed(scriptId)) { - apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); - return doInvokeFunction(scriptId, functionName, args); + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) { + String functionName = scriptIdToNameMap.get(scriptId); + if (functionName == null) { + return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); + } + if (!isDisabled(scriptId)) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); + return doInvokeFunction(scriptId, functionName, args); + } else { + return Futures.immediateFailedFuture( + new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!")); + } } else { - return Futures.immediateFailedFuture( - new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!")); + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); } } @@ -86,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { if (functionName != null) { try { scriptIdToNameMap.remove(scriptId); - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); doRelease(scriptId, functionName); } catch (Exception e) { return Futures.immediateFailedFuture(e); @@ -106,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { @@ -116,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } - private boolean isBlackListed(UUID scriptId) { - BlackListInfo errorCount = blackListedFunctions.get(scriptId); + private boolean isDisabled(UUID scriptId) { + DisableListInfo errorCount = disabledFunctions.get(scriptId); if (errorCount != null) { if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { - blackListedFunctions.remove(scriptId); + disabledFunctions.remove(scriptId); return false; } else { return errorCount.get() >= getMaxErrors(); @@ -130,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { } } - private class BlackListInfo { + private class DisableListInfo { private final AtomicInteger counter; private long expirationTime; - private BlackListInfo() { + private DisableListInfo() { this.counter = new AtomicInteger(0); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 17be0602f8..db8bb68b4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes; import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -65,8 +65,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @Value("${js.local.stats.enabled:false}") private boolean statsEnabled; - public AbstractNashornJsInvokeService(TbUsageStatsClient apiUsageStatsClient, JsExecutorService jsExecutor) { - super(apiUsageStatsClient); + public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient); this.jsExecutor = jsExecutor; } diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index c20f3e8c08..0dd45d340c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -19,7 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import java.util.concurrent.TimeUnit; @@ -43,8 +44,8 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; - public NashornJsInvokeService(TbUsageStatsClient apiUsageStatsClient, JsExecutorService jsExecutor) { - super(apiUsageStatsClient, jsExecutor); + public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) { + super(apiUsageStateService, apiUsageClient, jsExecutor); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 2ada577287..7b4114dce6 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -30,7 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -69,8 +70,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); - public RemoteJsInvokeService(TbUsageStatsClient apiUsageStatsClient) { - super(apiUsageStatsClient); + public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) { + super(apiUsageStateService, apiUsageClient); } @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java index 0048e325f3..b1ff230e71 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -15,12 +15,24 @@ */ package org.thingsboard.server.common.data; +import lombok.Getter; + public enum ApiUsageRecordKey { - TRANSPORT_MSG_COUNT, - TRANSPORT_DP_COUNT, - STORAGE_DP_COUNT, - RE_EXEC_COUNT, - JS_EXEC_COUNT + TRANSPORT_MSG_COUNT("transportMsgCount", "transportMsgLimit"), + TRANSPORT_DP_COUNT("transportDataPointsCount", "transportDataPointsLimit"), + STORAGE_DP_COUNT("storageDataPointsCount", "storageDataPointsLimit"), + RE_EXEC_COUNT("ruleEngineExecutionCount", "ruleEngineExecutionLimit"), + JS_EXEC_COUNT("jsExecutionCount", "jsExecutionLimit"); + + @Getter + private final String apiCountKey; + @Getter + private final String apiLimitKey; + + ApiUsageRecordKey(String apiCountKey, String apiLimitKey) { + this.apiCountKey = apiCountKey; + this.apiLimitKey = apiLimitKey; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index f361ec901c..65940916b3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.tenant.profile; import lombok.Data; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.TenantProfileType; @Data @@ -38,6 +39,24 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxDPStorageDays; private int maxRuleNodeExecutionsPerMessage; + @Override + public long getProfileThreshold(ApiUsageRecordKey key) { + switch (key) { + case TRANSPORT_MSG_COUNT: + return maxTransportMessages; + case TRANSPORT_DP_COUNT: + return maxTransportDataPoints; + case JS_EXEC_COUNT: + return maxJSExecutions; + case RE_EXEC_COUNT: + return maxREExecutions; + case STORAGE_DP_COUNT: + return maxDPStorageDays; + } + return 0L; + } + + @Override public TenantProfileType getType() { return TenantProfileType.DEFAULT; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java index 07b184b389..d058d11cfc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.TenantProfileType; @JsonIgnoreProperties(ignoreUnknown = true) @@ -33,4 +34,7 @@ public interface TenantProfileConfiguration { @JsonIgnore TenantProfileType getType(); + @JsonIgnore + long getProfileThreshold(ApiUsageRecordKey key); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java rename to common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java index 5a94cd505e..0702322838 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbUsageStatsClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageClient.java @@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @Slf4j -public class DefaultTbUsageStatsClient implements TbUsageStatsClient { +public class DefaultTbApiUsageClient implements TbApiUsageClient { @Value("${usage.stats.report.enabled:true}") private boolean enabled; @@ -53,7 +53,7 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { private final TbQueueProducerProvider producerProvider; private TbQueueProducer> msgProducer; - public DefaultTbUsageStatsClient(PartitionService partitionService, SchedulerComponent scheduler, TbQueueProducerProvider producerProvider) { + public DefaultTbApiUsageClient(PartitionService partitionService, SchedulerComponent scheduler, TbQueueProducerProvider producerProvider) { this.partitionService = partitionService; this.scheduler = scheduler; this.producerProvider = producerProvider; @@ -109,4 +109,8 @@ public class DefaultTbUsageStatsClient implements TbUsageStatsClient { } } + @Override + public void report(TenantId tenantId, ApiUsageRecordKey key) { + report(tenantId, key, 1L); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java rename to common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java index 4ed850a018..31d85b27c7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbUsageStatsClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/TbApiUsageClient.java @@ -18,9 +18,10 @@ package org.thingsboard.server.queue.usagestats; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.id.TenantId; -public interface TbUsageStatsClient { +public interface TbApiUsageClient { void report(TenantId tenantId, ApiUsageRecordKey key, long value); + void report(TenantId tenantId, ApiUsageRecordKey key); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index f77a3ce44b..29bc1b04a7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -78,7 +78,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.queue.scheduler.SchedulerComponent; -import org.thingsboard.server.queue.usagestats.TbUsageStatsClient; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.queue.util.TbTransportComponent; import javax.annotation.PostConstruct; @@ -123,7 +123,7 @@ public class DefaultTransportService implements TransportService { private final StatsFactory statsFactory; private final TransportDeviceProfileCache deviceProfileCache; private final TransportTenantProfileCache tenantProfileCache; - private final TbUsageStatsClient apiUsageStatsClient; + private final TbApiUsageClient apiUsageClient; private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; private final SchedulerComponent scheduler; @@ -152,7 +152,7 @@ public class DefaultTransportService implements TransportService { StatsFactory statsFactory, TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, - TbUsageStatsClient apiUsageStatsClient, TransportRateLimitService rateLimitService, + TbApiUsageClient apiUsageClient, TransportRateLimitService rateLimitService, DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; @@ -161,7 +161,7 @@ public class DefaultTransportService implements TransportService { this.statsFactory = statsFactory; this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; - this.apiUsageStatsClient = apiUsageStatsClient; + this.apiUsageClient = apiUsageClient; this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; this.scheduler = scheduler; @@ -647,6 +647,7 @@ public class DefaultTransportService implements TransportService { if (stateOpt.isPresent()) { ApiUsageState apiUsageState = stateOpt.get(); rateLimitService.update(apiUsageState.getTenantId(), apiUsageState.isTransportEnabled()); + //TODO: if transport is disabled, we should close all sessions and not to check credentials. } } } else if (toSessionMsg.hasEntityDeleteMsg()) { @@ -821,8 +822,8 @@ public class DefaultTransportService implements TransportService { @Override public void onSuccess(T msg) { try { - apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_MSG_COUNT, 1); - apiUsageStatsClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_DP_COUNT, dataPoints); + apiUsageClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_MSG_COUNT, 1); + apiUsageClient.report(tenantId, ApiUsageRecordKey.TRANSPORT_DP_COUNT, dataPoints); } finally { callback.onSuccess(msg); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java index 9c0687ab64..0deae8f729 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java @@ -51,4 +51,5 @@ public interface TenantProfileRepository extends PagingAndSortingRepository { TenantProfile findDefaultTenantProfile(TenantId tenantId); EntityInfo findDefaultTenantProfileInfo(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 15be73a0b8..e1b494e2e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -17,15 +17,26 @@ package org.thingsboard.server.dao.usagerecord; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantProfileDao; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.ArrayList; +import java.util.List; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -35,11 +46,15 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private final ApiUsageStateDao apiUsageStateDao; + private final TenantProfileDao tenantProfileDao; private final TenantDao tenantDao; + private final TimeseriesService tsService; - public ApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao) { + public ApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, TimeseriesService tsService) { this.tenantDao = tenantDao; this.apiUsageStateDao = apiUsageStateDao; + this.tenantProfileDao = tenantProfileDao; + this.tsService = tsService; } @Override @@ -57,7 +72,18 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A apiUsageState.setTenantId(tenantId); apiUsageState.setEntityId(tenantId); apiUsageStateValidator.validate(apiUsageState, ApiUsageState::getTenantId); - return apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + + ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + + Tenant tenant = tenantDao.findById(tenantId, tenantId.getId()); + TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenant.getTenantProfileId().getId()); + TenantProfileConfiguration configuration = tenantProfile.getProfileData().getConfiguration(); + List profileThresholds = new ArrayList<>(); + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + profileThresholds.add(new BasicTsKvEntry(saved.getCreatedTime(), new LongDataEntry(key.getApiLimitKey(), configuration.getProfileThreshold(key)))); + } + tsService.save(tenantId, saved.getId(), profileThresholds, 0L); + return saved; } @Override @@ -80,7 +106,8 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A log.trace("Executing findApiUsageStateById, tenantId [{}], apiUsageStateId [{}]", tenantId, id); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); validateId(id, "Incorrect apiUsageStateId " + id); - return apiUsageStateDao.findById(tenantId, id.getId()); } + return apiUsageStateDao.findById(tenantId, id.getId()); + } private DataValidator apiUsageStateValidator = new DataValidator() { From e9592174c2293b842ef6f1946dbd11d2e27056e4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 27 Oct 2020 19:39:14 +0200 Subject: [PATCH 19/30] Api Tenant States --- .../DefaultTbApiUsageStateService.java | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 1b1a2853c7..17563bcbbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,6 +23,7 @@ import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,11 +32,13 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -69,17 +72,14 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; public static final FutureCallback VOID_CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { - - } + public void onSuccess(@Nullable Void result) {} @Override - public void onFailure(Throwable t) { - - } + public void onFailure(Throwable t) {} }; private final TbClusterService clusterService; private final PartitionService partitionService; + private final TenantService tenantService; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; private final TelemetrySubscriptionService tsWsService; @@ -101,12 +101,13 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public DefaultTbApiUsageStateService(TbClusterService clusterService, PartitionService partitionService, - ApiUsageStateService apiUsageStateService, + TenantService tenantService, ApiUsageStateService apiUsageStateService, TimeseriesService tsService, TelemetrySubscriptionService tsWsService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { this.clusterService = clusterService; this.partitionService = partitionService; + this.tenantService = tenantService; this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; this.tsWsService = tsWsService; @@ -117,6 +118,8 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { @PostConstruct public void init() { if (enabled) { + log.info("Starting api usage service."); + initStatesFromDataBase(); scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS); } } @@ -164,6 +167,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); + initStatesFromDataBase(); } } @@ -299,12 +303,10 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { for (TsKvEntry tsKvEntry : dbValues) { if (tsKvEntry.getKey().equals(key.getApiCountKey())) { cycleEntryFound = true; - tenantState.put(key, tsKvEntry.getLongValue().get()); + tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L); } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { hourlyEntryFound = true; - if (tsKvEntry.getTs() == tenantState.getCurrentHourTs()) { - tenantState.putHourly(key, tsKvEntry.getLongValue().get()); - } + tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L); } if (cycleEntryFound && hourlyEntryFound) { break; @@ -319,4 +321,25 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { return tenantState; } + private void initStatesFromDataBase() { + try { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); + for (Tenant tenant : tenantIterator) { + if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { + updateLock.lock(); + try { + updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); + } catch (Exception e) { + log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); + } finally { + updateLock.unlock(); + } + } + } + log.info("Api usage service started."); + } catch (Exception e) { + log.warn("Unknown failure", e); + } + } + } From d4ae02f9a6dd94c1cad7adb5773fd94ea7ad374f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 27 Oct 2020 19:39:45 +0200 Subject: [PATCH 20/30] Update State of the API on startup --- .../server/service/apiusage/DefaultTbApiUsageStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 17563bcbbd..3f9fab11ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From feb9120050d3aaddccf0248f98d8ac6868cb4a40 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 28 Oct 2020 19:02:56 +0200 Subject: [PATCH 21/30] Improvements to Device Profile Rule Node --- .../actors/ruleChain/DefaultTbContext.java | 6 ++- .../server/controller/DeviceController.java | 3 ++ .../profile/DefaultTbDeviceProfileCache.java | 54 ++++++++++++------- .../service/profile/TbDeviceProfileCache.java | 4 +- .../processing/AbstractConsumerService.java | 2 +- .../api/RuleEngineDeviceProfileCache.java | 3 +- .../rule/engine/api/TbContext.java | 4 +- .../rule/engine/profile/AlarmState.java | 1 + .../engine/profile/TbDeviceProfileNode.java | 31 ++++++++++- 9 files changed, 80 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index e7a54385a7..ca401596f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import java.util.Collections; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -479,8 +481,8 @@ class DefaultTbContext implements TbContext { } @Override - public void addProfileListener(Consumer listener) { - mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), listener); + public void addDeviceProfileListeners(Consumer profileListener, BiConsumer deviceListener) { + mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index c8fde63c4d..6bb92648cf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; @@ -119,6 +120,8 @@ public class DeviceController extends BaseController { tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); + tbClusterService.onEntityStateChange(device.getTenantId(), device.getId(), + device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), diff --git a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java index 6784405adb..20dcb47d1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java @@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; import java.util.function.Consumer; @Service @@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { private final ConcurrentMap deviceProfilesMap = new ConcurrentHashMap<>(); private final ConcurrentMap devicesMap = new ConcurrentHashMap<>(); - private final ConcurrentMap>> listeners = new ConcurrentHashMap<>(); + private final ConcurrentMap>> profileListeners = new ConcurrentHashMap<>(); + private final ConcurrentMap>> deviceProfileListeners = new ConcurrentHashMap<>(); public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { this.deviceProfileService = deviceProfileService; @@ -87,33 +88,37 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { return get(tenantId, profileId); } - @Override - public void put(DeviceProfile profile) { - if (profile.getId() != null) { - deviceProfilesMap.put(profile.getId(), profile); - log.debug("[{}] pushed device profile to cache: {}", profile.getId(), profile); - notifyListeners(profile); - } - } - @Override public void evict(TenantId tenantId, DeviceProfileId profileId) { DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); DeviceProfile newProfile = get(tenantId, profileId); if (newProfile != null) { - notifyListeners(newProfile); + notifyProfileListeners(newProfile); } } @Override - public void evict(DeviceId deviceId) { - devicesMap.remove(deviceId); + public void evict(TenantId tenantId, DeviceId deviceId) { + DeviceProfileId old = devicesMap.remove(deviceId); + if (old != null) { + DeviceProfile newProfile = get(tenantId, deviceId); + if (newProfile == null || !old.equals(newProfile.getId())) { + notifyDeviceListeners(deviceId, newProfile); + } + } } @Override - public void addListener(TenantId tenantId, EntityId listenerId, Consumer listener) { - listeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, listener); + public void addListener(TenantId tenantId, EntityId listenerId, + Consumer profileListener, + BiConsumer deviceListener) { + if (profileListener != null) { + profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener); + } + if (deviceListener != null) { + deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener); + } } @Override @@ -128,17 +133,28 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { @Override public void removeListener(TenantId tenantId, EntityId listenerId) { - ConcurrentMap> tenantListeners = listeners.get(tenantId); + ConcurrentMap> tenantListeners = profileListeners.get(tenantId); if (tenantListeners != null) { tenantListeners.remove(listenerId); } + ConcurrentMap> deviceListeners = deviceProfileListeners.get(tenantId); + if (deviceListeners != null) { + deviceListeners.remove(listenerId); + } } - private void notifyListeners(DeviceProfile profile) { - ConcurrentMap> tenantListeners = listeners.get(profile.getTenantId()); + private void notifyProfileListeners(DeviceProfile profile) { + ConcurrentMap> tenantListeners = profileListeners.get(profile.getTenantId()); if (tenantListeners != null) { tenantListeners.forEach((id, listener) -> listener.accept(profile)); } } + private void notifyDeviceListeners(DeviceId deviceId, DeviceProfile profile) { + ConcurrentMap> tenantListeners = deviceProfileListeners.get(profile.getTenantId()); + if (tenantListeners != null) { + tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile)); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java index e65f297d53..10e28bd299 100644 --- a/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java @@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { - void put(DeviceProfile profile); - void evict(TenantId tenantId, DeviceProfileId id); - void evict(DeviceId id); + void evict(TenantId tenantId, DeviceId id); DeviceProfile find(DeviceProfileId deviceProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 9a4d488e0f..80bb6acd1d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -165,7 +165,7 @@ public abstract class AbstractConsumerService listener); + void addListener(TenantId tenantId, EntityId listenerId, Consumer profileListener, BiConsumer devicelistener); void removeListener(TenantId tenantId, EntityId listenerId); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 3d45fcf445..24af4f0d22 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -47,6 +48,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -225,7 +227,7 @@ public interface TbContext { void clearRuleNodeStates(); - void addProfileListener(Consumer listener); + void addDeviceProfileListeners(Consumer listener, BiConsumer deviceListener); void removeProfileListener(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java index df46a4bf88..6c771f0c86 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java @@ -103,6 +103,7 @@ class AlarmState { } AlarmEvalResult evalResult = evalFunction.apply(clearState, data); if (AlarmEvalResult.TRUE.equals(evalResult)) { + clearState.clear(); stateUpdate |= clearState.checkUpdate(); for (AlarmRuleState state : createRulesSortedBySeverityDesc) { state.clear(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java index b4bbf08323..0766265368 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java @@ -15,6 +15,8 @@ */ package org.thingsboard.rule.engine.profile; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.rule.engine.api.RuleNode; @@ -59,6 +61,7 @@ import java.util.concurrent.TimeUnit; public class TbDeviceProfileNode implements TbNode { private static final String PERIODIC_MSG_TYPE = "TbDeviceProfilePeriodicMsg"; private static final String PROFILE_UPDATE_MSG_TYPE = "TbDeviceProfileUpdateMsg"; + private static final String DEVICE_UPDATE_MSG_TYPE = "TbDeviceUpdateMsg"; private TbDeviceProfileNodeConfiguration config; private RuleEngineDeviceProfileCache cache; @@ -71,7 +74,7 @@ public class TbDeviceProfileNode implements TbNode { this.cache = ctx.getDeviceProfileCache(); this.ctx = ctx; scheduleAlarmHarvesting(ctx); - ctx.addProfileListener(this::onProfileUpdate); + ctx.addDeviceProfileListeners(this::onProfileUpdate, this::onDeviceUpdate); if (config.isFetchAlarmRulesStateOnStart()) { log.info("[{}] Fetching alarm rule state", ctx.getSelfId()); int fetchCount = 0; @@ -108,6 +111,15 @@ public class TbDeviceProfileNode implements TbNode { harvestAlarms(ctx, System.currentTimeMillis()); } else if (msg.getType().equals(PROFILE_UPDATE_MSG_TYPE)) { updateProfile(ctx, new DeviceProfileId(UUID.fromString(msg.getData()))); + } else if (msg.getType().equals(DEVICE_UPDATE_MSG_TYPE)) { + JsonNode data = JacksonUtil.toJsonNode(msg.getData()); + DeviceId deviceId = new DeviceId(UUID.fromString(data.get("deviceId").asText())); + if (data.has("profileId")) { + invalidateDeviceProfileCache(deviceId, new DeviceProfileId(UUID.fromString(data.get("deviceProfileId").asText()))); + } else { + deviceStates.remove(deviceId); + } + } else { if (EntityType.DEVICE.equals(originatorType)) { DeviceId deviceId = new DeviceId(msg.getOriginator().getId()); @@ -182,6 +194,15 @@ public class TbDeviceProfileNode implements TbNode { ctx.tellSelf(TbMsg.newMsg(PROFILE_UPDATE_MSG_TYPE, ctx.getTenantId(), TbMsgMetaData.EMPTY, profile.getId().getId().toString()), 0L); } + private void onDeviceUpdate(DeviceId deviceId, DeviceProfile deviceProfile) { + ObjectNode msgData = JacksonUtil.newObjectNode(); + msgData.put("deviceId", deviceId.getId().toString()); + if (deviceProfile != null) { + msgData.put("deviceProfileId", deviceProfile.getId().getId().toString()); + } + ctx.tellSelf(TbMsg.newMsg(DEVICE_UPDATE_MSG_TYPE, ctx.getTenantId(), TbMsgMetaData.EMPTY, JacksonUtil.toString(msgData)), 0L); + } + protected void invalidateDeviceProfileCache(DeviceId deviceId, String deviceJson) { DeviceState deviceState = deviceStates.get(deviceId); if (deviceState != null) { @@ -193,4 +214,12 @@ public class TbDeviceProfileNode implements TbNode { } } + protected void invalidateDeviceProfileCache(DeviceId deviceId, DeviceProfileId deviceProfileId) { + DeviceState deviceState = deviceStates.get(deviceId); + if (deviceState != null) { + if (!deviceState.getProfileId().equals(deviceProfileId)) { + deviceStates.remove(deviceId); + } + } + } } From 78ac876103ff675c4949a1a15c8b4aecc4786e14 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 29 Oct 2020 13:26:15 +0200 Subject: [PATCH 22/30] Bug fixes for Device Profile node --- .../server/actors/ruleChain/DefaultTbContext.java | 8 ++++++++ .../server/dao/rule/RuleNodeStateService.java | 2 ++ .../data/device/profile/DeviceProfileAlarm.java | 3 ++- .../server/dao/rule/BaseRuleNodeStateService.java | 14 ++++++++++++++ .../server/dao/rule/RuleNodeStateDao.java | 2 ++ .../server/dao/sql/rule/JpaRuleNodeStateDao.java | 6 ++++++ .../dao/sql/rule/RuleNodeStateRepository.java | 1 + .../thingsboard/rule/engine/api/TbContext.java | 2 ++ .../rule/engine/profile/TbDeviceProfileNode.java | 15 +++++++++++---- .../engine/profile/TbDeviceProfileNodeTest.java | 3 ++- 10 files changed, 50 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ca401596f6..72177c02f2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -480,6 +480,14 @@ class DefaultTbContext implements TbContext { mainCtx.getRuleNodeStateService().removeByRuleNodeId(getTenantId(), getSelfId()); } + @Override + public void removeRuleNodeStateForEntity(EntityId entityId) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId); + } + mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId); + } + @Override public void addDeviceProfileListeners(Consumer profileListener, BiConsumer deviceListener) { mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java index d5ad9bbbb6..924cad0884 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java @@ -31,4 +31,6 @@ public interface RuleNodeStateService { RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); + + void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java index b6437ae7bb..17035f337b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +28,7 @@ public class DeviceProfileAlarm { private String id; private String alarmType; - private Map createRules; + private LinkedHashMap createRules; private AlarmRule clearRule; // Hidden in advanced settings diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java index 1d7e8af03f..7bc77e14a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleNodeStateService.java @@ -79,6 +79,20 @@ public class BaseRuleNodeStateService extends AbstractEntityService implements R ruleNodeStateDao.removeByRuleNodeId(ruleNodeId.getId()); } + @Override + public void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId ruleNodeId, EntityId entityId) { + if (tenantId == null) { + throw new DataValidationException("Tenant id should be specified!."); + } + if (ruleNodeId == null) { + throw new DataValidationException("Rule node id should be specified!."); + } + if (entityId == null) { + throw new DataValidationException("Entity id should be specified!."); + } + ruleNodeStateDao.removeByRuleNodeIdAndEntityId(ruleNodeId.getId(), entityId.getId()); + } + public RuleNodeState saveOrUpdate(TenantId tenantId, RuleNodeState ruleNodeState, boolean update) { try { if (update) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java index 89c2baea7f..9f304dd828 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java @@ -35,4 +35,6 @@ public interface RuleNodeStateDao extends Dao { RuleNodeState findByRuleNodeIdAndEntityId(UUID ruleNodeId, UUID entityId); void removeByRuleNodeId(UUID ruleNodeId); + + void removeByRuleNodeIdAndEntityId(UUID ruleNodeId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java index c61e58353c..e32f90a645 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java @@ -63,4 +63,10 @@ public class JpaRuleNodeStateDao extends JpaAbstractDao(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); KeyFilter lowTempFilter = new KeyFilter(); lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); From 3ee3839c3c00cd18bf729515530950235fb8edab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 29 Oct 2020 15:53:11 +0200 Subject: [PATCH 23/30] Protection from infinite loops --- .../RuleChainActorMessageProcessor.java | 1 - .../RuleNodeActorMessageProcessor.java | 48 +++++++++++------ .../actors/shared/ComponentMsgProcessor.java | 6 +++ .../DefaultTenantProfileConfiguration.java | 5 ++ .../profile/TenantProfileConfiguration.java | 3 ++ .../tenant/profile/TenantProfileData.java | 6 --- .../thingsboard/server/common/msg/TbMsg.java | 54 ++++++++++++------- common/message/src/main/proto/tbmsg.proto | 1 + 8 files changed, 82 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index f5df62cfd3..ad5f97ab48 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -334,7 +334,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor extends Abstract this.entityId = id; } + protected TenantProfileConfiguration getTenantProfileConfiguration() { + return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration(); + } + public abstract String getComponentName(); public abstract void start(TbActorCtx context) throws Exception; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 65940916b3..1804731c04 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -61,4 +61,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura public TenantProfileType getType() { return TenantProfileType.DEFAULT; } + + @Override + public int getMaxRuleNodeExecsPerMessage() { + return maxRuleNodeExecutionsPerMessage; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java index d058d11cfc..c4c6d7563a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java @@ -37,4 +37,7 @@ public interface TenantProfileConfiguration { @JsonIgnore long getProfileThreshold(ApiUsageRecordKey key); + @JsonIgnore + int getMaxRuleNodeExecsPerMessage(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index 0996fc9850..297e562d26 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -15,14 +15,8 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import java.util.HashMap; -import java.util.Map; - @Data public class TenantProfileData { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index f0af529be9..3b00e89fd6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import lombok.AccessLevel; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import java.io.IOException; import java.io.Serializable; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by ashvayka on 13.01.18. */ @Data -@Builder @Slf4j public final class TbMsg implements Serializable { @@ -52,51 +52,63 @@ public final class TbMsg implements Serializable { private final String data; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; + @Getter(value = AccessLevel.NONE) + private final AtomicInteger ruleNodeExecCounter; + + public int getAndIncrementRuleNodeCounter() { + return ruleNodeExecCounter.getAndIncrement(); + } + //This field is not serialized because we use queues and there is no need to do it @JsonIgnore transient private final TbMsgCallback callback; public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, + metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } + // REALLY NEW MSG + public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY); } + // For Tests only + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY); } public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback); } - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(), + data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } - public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) { - return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, - origMsg.data, ruleChainId, null, origMsg.getCallback()); + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) { + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType, + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback()); } public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY); } private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) { this.id = id; this.queueName = queueName; if (ts > 0) { @@ -111,6 +123,7 @@ public final class TbMsg implements Serializable { this.data = data; this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; + this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter); if (callback != null) { this.callback = callback; } else { @@ -147,6 +160,7 @@ public final class TbMsg implements Serializable { builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); + builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get()); return builder.build().toByteArray(); } @@ -164,18 +178,18 @@ public final class TbMsg implements Serializable { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback); } public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback); } public TbMsgCallback getCallback() { diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 25e006121f..53cb90014b 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -45,4 +45,5 @@ message TbMsgProto { string data = 14; int64 ts = 15; + int32 ruleNodeExecCounter = 16; } \ No newline at end of file From 32f279263d1652000021a02b45d178bd563a7bcb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 2 Nov 2020 13:13:12 +0200 Subject: [PATCH 24/30] Fix for entity events --- .../org/thingsboard/server/controller/DeviceController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 6bb92648cf..5171d9f7c1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -120,7 +120,7 @@ public class DeviceController extends BaseController { tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); - tbClusterService.onEntityStateChange(device.getTenantId(), device.getId(), + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), device.getId(), device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedDevice.getId(), savedDevice, From 07c1e680fe102dfc5aa1af79b94d5885d31859e2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 2 Nov 2020 16:40:49 +0200 Subject: [PATCH 25/30] Build fix --- .../org/thingsboard/server/controller/DeviceController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 5171d9f7c1..03ccc9fb93 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -120,7 +120,7 @@ public class DeviceController extends BaseController { tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); - tbClusterService.onEntityStateChange(savedDevice.getTenantId(), device.getId(), + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(), device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedDevice.getId(), savedDevice, From 6d5d9a55c9b119d7a2e70793672b291f0b49ebe6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 2 Nov 2020 19:38:23 +0200 Subject: [PATCH 26/30] Minor security improvements --- .../controller/RuleChainController.java | 2 +- .../DefaultTbApiUsageStateService.java | 3 +- .../DefaultTelemetrySubscriptionService.java | 37 ++++++++++++++++ .../telemetry/InternalTelemetryService.java | 44 +++++++++++++++++++ .../TelemetrySubscriptionService.java | 2 +- 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 2ca834d85b..7392f74231 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -160,7 +160,7 @@ public class RuleChainController extends BaseController { public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { try { checkNotNull(request); - checkNotNull(request.getName()); + checkParameter(request.getName(), "name"); RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 3f9fab11ec..02d6e88284 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -50,6 +50,7 @@ import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.telemetry.InternalTelemetryService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.PostConstruct; @@ -82,7 +83,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { private final TenantService tenantService; private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; - private final TelemetrySubscriptionService tsWsService; + private final InternalTelemetryService tsWsService; private final SchedulerComponent scheduler; private final TbTenantProfileCache tenantProfileCache; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index aecd888dce..d4786a6f54 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -116,6 +116,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { + checkInternalEntity(entityId); + saveAndNotifyInternal(tenantId, entityId, ts, ttl, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); @@ -176,6 +182,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { + checkInternalEntity(entityId); + saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback); + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); @@ -183,6 +195,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + checkInternalEntity(entityId); + saveLatestAndNotifyInternal(tenantId, entityId, ts, callback); + } + + @Override + public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); @@ -190,6 +208,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback); + } + + @Override + public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); addMainCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); @@ -197,6 +221,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteLatest(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteLatestInternal(tenantId, entityId, keys, callback); + } + + @Override + public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); addMainCallback(deleteFuture, callback); } @@ -296,4 +326,11 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } }, tsCallBackExecutor); } + + private void checkInternalEntity(EntityId entityId) { + if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) { + throw new RuntimeException("Can't update API Usage State!"); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java new file mode 100644 index 0000000000..275630a161 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +/** + * Created by ashvayka on 27.03.18. + */ +public interface InternalTelemetryService extends RuleEngineTelemetryService { + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); + + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); + + void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + + void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback); + + void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback); + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d652a3f1c5..a88a378cd7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { +public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener { } From a64c4e3e203e1854c0ced8349dfb56e948a24ded Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Nov 2020 13:05:31 +0200 Subject: [PATCH 27/30] DeviceProfileNode fix for conditions based on the attributes --- .../service/apiusage/DefaultTbApiUsageStateService.java | 6 +++--- .../controller/BaseTenantProfileControllerTest.java | 5 ++++- .../server/controller/ControllerSqlTestSuite.java | 2 +- .../server/dao/tenant/TenantProfileServiceImpl.java | 8 +++++++- .../server/dao/usagerecord/ApiUsageStateServiceImpl.java | 2 +- .../thingsboard/rule/engine/profile/AlarmRuleState.java | 4 ++++ 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 02d6e88284..f7e00b7242 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -156,7 +156,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } finally { updateLock.unlock(); } - tsWsService.saveAndNotify(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L, VOID_CALLBACK); if (!result.isEmpty()) { persistAndNotify(tenantState, result); } @@ -256,7 +256,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } if (!profileThresholds.isEmpty()) { - tsWsService.saveAndNotify(tenantId, id, profileThresholds, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, 0L, VOID_CALLBACK); } } @@ -266,7 +266,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { long ts = System.currentTimeMillis(); List stateTelemetry = new ArrayList<>(); result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState))))); - tsWsService.saveAndNotify(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, 0L, VOID_CALLBACK); } private void checkStartOfNextCycle() { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index 311a93bf82..4f0d1e201b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName(name); tenantProfile.setDescription(name + " Test"); - tenantProfile.setProfileData(new TenantProfileData()); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); tenantProfile.setDefault(false); tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbRuleEngine(false); diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 64f9b8fddf..bc2c36b501 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -27,7 +27,7 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", -// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", +// "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest", "org.thingsboard.server.controller.sql.*Test", }) public class ControllerSqlTestSuite { diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index af327d3505..5bafe4f799 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -216,6 +216,12 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T if (StringUtils.isEmpty(tenantProfile.getName())) { throw new DataValidationException("Tenant profile name should be specified!"); } + if (tenantProfile.getProfileData() == null) { + throw new DataValidationException("Tenant profile data should be specified!"); + } + if (tenantProfile.getProfileData().getConfiguration() == null) { + throw new DataValidationException("Tenant profile data configuration should be specified!"); + } if (tenantProfile.isDefault()) { TenantProfile defaultTenantProfile = findDefaultTenantProfile(tenantId); if (defaultTenantProfile != null && !defaultTenantProfile.getId().equals(tenantProfile.getId())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index e1b494e2e8..26184a5824 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 8be2b81a59..b1eeb82b15 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -99,6 +99,10 @@ class AlarmRuleState { } } for (EntityKey key : changedKeys) { + EntityKeyType keyType = key.getType(); + if (EntityKeyType.CLIENT_ATTRIBUTE.equals(keyType) || EntityKeyType.SERVER_ATTRIBUTE.equals(keyType) || EntityKeyType.SHARED_ATTRIBUTE.equals(keyType)) { + key = new EntityKey(EntityKeyType.ATTRIBUTE, key.getKey()); + } if (entityKeys.contains(key)) { return true; } From 35dbe509f071d94fb1cf53fc637a3a8e4ff5e36f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Nov 2020 13:43:27 +0200 Subject: [PATCH 28/30] Data Points storage --- .../DefaultTbApiUsageStateService.java | 4 +- .../AbstractSubscriptionService.java | 6 +-- .../DefaultTelemetrySubscriptionService.java | 40 +++++++++++-------- .../telemetry/InternalTelemetryService.java | 2 +- .../dao/timeseries/TimeseriesService.java | 4 +- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index f7e00b7242..88868d2258 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -71,9 +71,9 @@ import java.util.concurrent.locks.ReentrantLock; public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; - public static final FutureCallback VOID_CALLBACK = new FutureCallback() { + public static final FutureCallback VOID_CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) {} + public void onSuccess(@Nullable Integer result) {} @Override public void onFailure(Throwable t) {} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index bc4099e81b..4178502c77 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } } - protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback>() { + protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback>() { @Override - public void onSuccess(@Nullable List result) { + public void onSuccess(@Nullable List result) { callback.accept(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d4786a6f54..53d0e08da2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.EntityId; @@ -42,10 +41,9 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.queue.TbClusterService; -import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; @@ -59,12 +57,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -76,6 +70,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final AttributesService attrService; private final TimeseriesService tsService; private final EntityViewService entityViewService; + private final TbApiUsageClient apiUsageClient; private ExecutorService tsCallBackExecutor; @@ -83,11 +78,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, EntityViewService entityViewService, TbClusterService clusterService, - PartitionService partitionService) { + PartitionService partitionService, + TbApiUsageClient apiUsageClient) { super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; this.entityViewService = entityViewService; + this.apiUsageClient = apiUsageClient; } @PostConstruct @@ -117,12 +114,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { checkInternalEntity(entityId); - saveAndNotifyInternal(tenantId, entityId, ts, ttl, callback); + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { + @Override + public void onSuccess(Integer result) { + if (result != null && result > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + } + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); } @Override - public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { + ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { @@ -147,9 +157,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Optional tsKvEntry = entries.stream() .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) .max(Comparator.comparingLong(TsKvEntry::getTs)); - if (tsKvEntry.isPresent()) { - entityViewLatest.add(tsKvEntry.get()); - } + tsKvEntry.ifPresent(entityViewLatest::add); } } if (!entityViewLatest.isEmpty()) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 275630a161..53cc23f23a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -29,7 +29,7 @@ import java.util.List; */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 2a5a701f9e..81dedff132 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -36,9 +36,9 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); From 123ea76c94718c4eaf251a738887727ad2564abb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Nov 2020 18:59:12 +0200 Subject: [PATCH 29/30] Improvements for data points storage --- .../DefaultTbApiUsageStateService.java | 30 ++++---- .../DefaultRuleEngineStatisticsService.java | 8 +-- .../AbstractSubscriptionService.java | 6 +- .../DefaultTelemetrySubscriptionService.java | 68 +++++++++++++------ .../telemetry/InternalTelemetryService.java | 2 + .../dao/timeseries/TimeseriesService.java | 4 +- .../server/common/data/kv/BasicTsKvEntry.java | 19 +++++- .../server/common/data/kv/TsKvEntry.java | 5 ++ ...stractChunkedAggregationTimeseriesDao.java | 2 +- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 21 ++++++ .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 7 +- .../dao/sqlts/psql/JpaPsqlTimeseriesDao.java | 7 +- .../timescale/TimescaleTimeseriesDao.java | 11 +-- .../dao/tenant/TenantProfileServiceImpl.java | 2 +- .../dao/timeseries/BaseTimeseriesService.java | 34 +++++++--- .../CassandraBaseTimeseriesDao.java | 19 ++++-- .../server/dao/timeseries/TimeseriesDao.java | 4 +- .../usagerecord/ApiUsageStateServiceImpl.java | 2 +- .../dao/service/BaseEntityServiceTest.java | 4 +- 19 files changed, 181 insertions(+), 74 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 88868d2258..42f980909a 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.apiusage; import com.google.common.util.concurrent.FutureCallback; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -51,7 +53,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.InternalTelemetryService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.PostConstruct; import java.util.ArrayList; @@ -73,20 +74,25 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; public static final FutureCallback VOID_CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Integer result) {} + public void onSuccess(@Nullable Integer result) { + } @Override - public void onFailure(Throwable t) {} + public void onFailure(Throwable t) { + } }; private final TbClusterService clusterService; private final PartitionService partitionService; private final TenantService tenantService; - private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; - private final InternalTelemetryService tsWsService; + private final ApiUsageStateService apiUsageStateService; private final SchedulerComponent scheduler; private final TbTenantProfileCache tenantProfileCache; + @Lazy + @Autowired + private InternalTelemetryService tsWsService; + // Tenants that should be processed on this server private final Map myTenantStates = new ConcurrentHashMap<>(); // Tenants that should be processed on other servers @@ -102,16 +108,16 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public DefaultTbApiUsageStateService(TbClusterService clusterService, PartitionService partitionService, - TenantService tenantService, ApiUsageStateService apiUsageStateService, - TimeseriesService tsService, TelemetrySubscriptionService tsWsService, + TenantService tenantService, + TimeseriesService tsService, + ApiUsageStateService apiUsageStateService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { this.clusterService = clusterService; this.partitionService = partitionService; this.tenantService = tenantService; - this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; - this.tsWsService = tsWsService; + this.apiUsageStateService = apiUsageStateService; this.scheduler = scheduler; this.tenantProfileCache = tenantProfileCache; } @@ -156,7 +162,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } finally { updateLock.unlock(); } - tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK); if (!result.isEmpty()) { persistAndNotify(tenantState, result); } @@ -256,7 +262,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } if (!profileThresholds.isEmpty()) { - tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK); } } @@ -266,7 +272,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { long ts = System.currentTimeMillis(); List stateTelemetry = new ArrayList<>(); result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState))))); - tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK); } private void checkStartOfNextCycle() { diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index b46033389b..a832eb0870 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -48,9 +48,9 @@ import java.util.stream.Collectors; public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; - public static final FutureCallback CALLBACK = new FutureCallback() { + public static final FutureCallback CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable Integer result) { } @@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .collect(Collectors.toList()); if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK); } } } catch (DataValidationException e) { @@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); } catch (DataValidationException e2) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { throw e2; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 4178502c77..468c3c26b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } } - protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback>() { + protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { @Override - public void onSuccess(@Nullable List result) { + public void onSuccess(@Nullable T result) { callback.accept(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 53d0e08da2..7a35e6b317 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,9 +40,11 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -71,6 +73,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TimeseriesService tsService; private final EntityViewService entityViewService; private final TbApiUsageClient apiUsageClient; + private final TbApiUsageStateService apiUsageStateService; private ExecutorService tsCallBackExecutor; @@ -79,12 +82,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityViewService entityViewService, TbClusterService clusterService, PartitionService partitionService, - TbApiUsageClient apiUsageClient) { + TbApiUsageClient apiUsageClient, + TbApiUsageStateService apiUsageStateService) { super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; this.entityViewService = entityViewService; this.apiUsageClient = apiUsageClient; + this.apiUsageStateService = apiUsageStateService; } @PostConstruct @@ -114,25 +119,34 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { checkInternalEntity(entityId); - saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { - @Override - public void onSuccess(Integer result) { - if (result != null && result > 0) { - apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { + @Override + public void onSuccess(Integer result) { + if (result != null && result > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + } + callback.onSuccess(null); } - callback.onSuccess(null); - } - @Override - public void onFailure(Throwable t) { - callback.onFailure(t); - } - }); + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } else{ + callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); + } + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback); } @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); + ListenableFuture saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { @@ -197,7 +211,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @@ -210,7 +224,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @@ -223,7 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); } @@ -236,7 +250,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); } @Override @@ -321,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + private void addVoidCallback(ListenableFuture saveFuture, final FutureCallback callback) { Futures.addCallback(saveFuture, new FutureCallback() { @Override public void onSuccess(@Nullable S result) { @@ -335,6 +349,20 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }, tsCallBackExecutor); } + private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable S result) { + callback.onSuccess(result); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }, tsCallBackExecutor); + } + private void checkInternalEntity(EntityId entityId) { if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) { throw new RuntimeException("Can't update API Usage State!"); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 53cc23f23a..1ca2cdeacb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -29,6 +29,8 @@ import java.util.List; */ public interface InternalTelemetryService extends RuleEngineTelemetryService { + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 81dedff132..0e11aedaf5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -36,9 +36,9 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index c2d6688004..bcf49899dc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -19,7 +19,7 @@ import java.util.Objects; import java.util.Optional; public class BasicTsKvEntry implements TsKvEntry { - + private static final int MAX_CHARS_PER_DATA_POINT = 512; private final long ts; private final KvEntry kv; @@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry { public String getValueAsString() { return kv.getValueAsString(); } + + @Override + public int getDataPoints() { + int length; + switch (getDataType()) { + case STRING: + length = getStrValue().get().length(); + break; + case JSON: + length = getJsonValue().get().length(); + break; + default: + return 1; + } + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index 315d3940df..e2f8eaf1e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * Represents time series KV data entry * @@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry { long getTs(); + @JsonIgnore + int getDataPoints(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 3cee730ee3..616260ba45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -100,7 +100,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { return Futures.immediateFuture(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 8dcefeca61..c585804b82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -30,11 +30,14 @@ import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { + protected static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + @Autowired protected ScheduledLogExecutorComponent logExecutor; @@ -56,6 +59,9 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries @Value("${sql.batch_sort:false}") protected boolean batchSortEnabled; + @Value("${sql.ttl.ts.ts_key_value_ttl:0}") + private long systemTtl; + protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() @@ -75,4 +81,19 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries } }, service); } + + protected long computeTtl(long ttl) { + if (systemTtl > 0) { + if (ttl == 0) { + ttl = systemTtl; + } else { + ttl = Math.min(systemTtl, ttl); + } + } + return ttl; + } + + protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { + return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java index 5ea387328d..025134e187 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.hsql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; @@ -35,7 +37,8 @@ import org.thingsboard.server.dao.util.SqlTsDao; public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao implements TimeseriesDao { @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TsKvEntity entity = new TsKvEntity(); @@ -48,7 +51,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 8c50bcb38c..3d971f61c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.psql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -71,7 +73,8 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); savePartitionIfNotExist(tsKvEntry.getTs()); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); @@ -85,7 +88,7 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } private void savePartitionIfNotExist(long ts) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 6d5923c508..f6fb027d7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -101,7 +102,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); @@ -113,14 +115,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(0); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 5bafe4f799..95dd0ab45a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index b86d6128f6..3b97e2bb82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.timeseries; +import com.google.common.base.Function; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -55,6 +57,20 @@ public class BaseTimeseriesService implements TimeseriesService { private static final int INSERTS_PER_ENTRY = 3; private static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY; + public static final Function, Integer> SUM_ALL_INTEGERS = new Function, Integer>() { + @Override + public @Nullable Integer apply(@Nullable List input) { + int result = 0; + if (input != null) { + for (Integer tmp : input) { + if (tmp != null) { + result += tmp; + } + } + } + return result; + } + }; @Value("${database.ts_max_intervals}") private long maxTsIntervals; @@ -101,26 +117,26 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } - List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); + List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); for (TsKvEntry tsKvEntry : tsKvEntries) { if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); } - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override @@ -135,12 +151,12 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.allAsList(futures); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey(), ttl)); - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } @@ -216,7 +232,7 @@ public class BaseTimeseriesService implements TimeseriesService { } else if (query.getAggregation() == null) { throw new IncorrectParameterException("Incorrect ReadTsKvQuery. Aggregation can't be empty"); } - if(!Aggregation.NONE.equals(query.getAggregation())) { + if (!Aggregation.NONE.equals(query.getAggregation())) { long step = Math.max(query.getInterval(), 1000); long intervalCounts = (query.getEndTs() - query.getStartTs()) / step; if (intervalCounts > maxTsIntervals || intervalCounts < 0) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 2f3fe4854c..b9d8c62833 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -60,6 +60,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -74,6 +75,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; + public static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); @Autowired @@ -141,9 +144,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { List> futures = new ArrayList<>(); ttl = computeTtl(ttl); + int dataPointDays = tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); long partition = toPartitionTs(tsKvEntry.getTs()); DataType type = tsKvEntry.getDataType(); if (setNullValuesEnabled) { @@ -161,11 +165,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } BoundStatement stmt = stmtBuilder.build(); futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + return Futures.transform(Futures.allAsList(futures), result -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { if (isFixedPartitioning()) { return Futures.immediateFuture(null); } @@ -181,7 +185,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD stmtBuilder.setInt(4, (int) ttl); } BoundStatement stmt = stmtBuilder.build(); - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> 0); } @Override @@ -649,9 +653,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } /** - // * Select existing partitions from the table - // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity - // */ + * // * Select existing partitions from the table + * // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity + * // + */ private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index dda2b227c3..2dd52580b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -31,9 +31,9 @@ public interface TimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); - ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); + ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 26184a5824..e1b494e2e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index d04d18772d..ff87e17d56 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -793,7 +793,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -1272,7 +1272,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); From 5f50fd44d2e6fbb53f4cab1a32bb52fc69748c35 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 9 Nov 2020 14:06:20 +0200 Subject: [PATCH 30/30] Usage Records implementation --- .../server/controller/TenantController.java | 2 ++ .../apiusage/DefaultTbApiUsageStateService.java | 9 +++++++-- .../thingsboard/server/common/data/TenantProfile.java | 11 +++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 83e3d98b1b..a42cbb0f91 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -93,6 +93,8 @@ public class TenantController extends BaseController { } tenantProfileCache.evict(tenant.getId()); tbClusterService.onTenantChange(tenant, null); + tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(), + newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); return tenant; } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 42f980909a..2c8879ba94 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -243,6 +243,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { TenantProfileData oldProfileData = state.getTenantProfileData(); + state.setTenantProfileId(tenantProfile.getId()); state.setTenantProfileData(tenantProfile.getProfileData()); Map result = state.checkStateUpdatedDueToThresholds(); if (!result.isEmpty()) { @@ -257,8 +258,10 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { long ts = System.currentTimeMillis(); List profileThresholds = new ArrayList<>(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { - if (oldData == null || oldData.getProfileThreshold(key) != newData.getProfileThreshold(key)) { - profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newData.getProfileThreshold(key)))); + long newProfileThreshold = newData.getProfileThreshold(key); + if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) { + log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold); + profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold))); } } if (!profileThresholds.isEmpty()) { @@ -267,6 +270,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } private void persistAndNotify(TenantApiUsageState state, Map result) { + log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result); apiUsageStateService.update(state.getApiUsageState()); clusterService.onApiStateChange(state.getApiUsageState(), null); long ts = System.currentTimeMillis(); @@ -320,6 +324,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } } + log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity); myTenantStates.put(tenantId, tenantState); } catch (InterruptedException | ExecutionException e) { log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 5e5f4a09de..acb9c239b8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -21,6 +21,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import java.io.ByteArrayInputStream; @@ -79,15 +80,21 @@ public class TenantProfile extends SearchTextBased implements H profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); } catch (IOException e) { log.warn("Can't deserialize tenant profile data: ", e); - return null; + return createDefaultTenantProfileData(); } return profileData; } else { - return null; + return createDefaultTenantProfileData(); } } } + public TenantProfileData createDefaultTenantProfileData() { + TenantProfileData tpd = new TenantProfileData(); + tpd.setConfiguration(new DefaultTenantProfileConfiguration()); + return tpd; + } + public void setProfileData(TenantProfileData data) { this.profileData = data; try {