From 1610222b72528ae7399bfc82167210f9171d5764 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 17:27:53 +0300 Subject: [PATCH] Export/import api rate limiting --- .../apiusage/DefaultRateLimitService.java | 76 +++++++++++++++++++ .../service/apiusage/RateLimitService.java | 26 +++++++ .../DefaultEntitiesExportImportService.java | 11 +++ .../DefaultTenantProfileConfiguration.java | 5 +- .../server/common/msg/tools/TbRateLimits.java | 8 +- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 777e84f348..859f3b9da2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.apiusage.RateLimitService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; @@ -55,6 +57,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); + private final RateLimitService rateLimitService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE @@ -63,6 +67,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); @@ -73,6 +81,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 15eb429805..9953f7a3aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index afcbe1b3b9..9ca13b295d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -27,6 +27,7 @@ import java.time.Duration; */ public class TbRateLimits { private final LocalBucket bucket; + private final String configuration; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); @@ -42,8 +43,7 @@ public class TbRateLimits { } else { throw new IllegalArgumentException("Failed to parse rate limits configuration: " + limitsConfiguration); } - - + this.configuration = limitsConfiguration; } public boolean tryConsume() { @@ -54,4 +54,8 @@ public class TbRateLimits { return bucket.tryConsume(number); } + public String getConfiguration() { + return configuration; + } + }