From 12951115f922312c2c49e8399657a9748345ae6b Mon Sep 17 00:00:00 2001 From: vparomskiy Date: Fri, 16 Feb 2018 14:22:55 +0200 Subject: [PATCH] Add white & black list. Fix after review --- .../src/main/resources/thingsboard.yml | 9 ++++++ .../inmemory/HostRequestIntervalRegistry.java | 25 ++++++++++++---- .../inmemory/IntervalRegistryLogger.java | 2 +- .../quota/HostRequestsQuotaServiceTest.java | 16 ++++++++-- .../HostRequestIntervalRegistryTest.java | 30 ++++++++++++++++++- .../transport/coap/CoapTransportResource.java | 1 - 6 files changed, 72 insertions(+), 11 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index da35c6c993..a2ae7e2a43 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -110,11 +110,20 @@ coap: #Quota parameters quota: host: + # Max allowed number of API requests in interval for single host limit: "${QUOTA_HOST_LIMIT:10000}" + # Interval duration intervalMs: "${QUOTA_HOST_INTERVAL_MS:60000}" + # Maximum silence duration for host after which Host removed from QuotaService. Must be bigger than intervalMs ttlMs: "${QUOTA_HOST_TTL_MS:60000}" + # Interval for scheduled task that cleans expired records. TTL is used for expiring cleanPeriodMs: "${QUOTA_HOST_CLEAN_PERIOD_MS:300000}" + # Enable Host API Limits enabled: "${QUOTA_HOST_ENABLED:false}" + # Array of whitelist hosts + whitelist: "${QUOTA_HOST_WHITELIST:localhost,127.0.0.1}" + # Array of blacklist hosts + blacklist: "${QUOTA_HOST_BLACKLIST:}" log: topSize: 10 intervalMin: 2 diff --git a/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java b/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java index 9f8f9465c2..4db6ff01b9 100644 --- a/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java +++ b/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2017 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,12 +15,15 @@ */ package org.thingsboard.server.common.transport.quota.inmemory; +import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -35,11 +38,17 @@ public class HostRequestIntervalRegistry { private final Map hostCounts = new ConcurrentHashMap<>(); private final long intervalDurationMs; private final long ttlMs; + private final Set whiteList; + private final Set blackList; public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs, - @Value("${quota.host.ttlMs}") long ttlMs) { + @Value("${quota.host.ttlMs}") long ttlMs, + @Value("${quota.host.whitelist}") String whiteList, + @Value("${quota.host.blacklist}") String blackList) { this.intervalDurationMs = intervalDurationMs; this.ttlMs = ttlMs; + this.whiteList = Sets.newHashSet(StringUtils.split(whiteList, ',')); + this.blackList = Sets.newHashSet(StringUtils.split(blackList, ',')); } @PostConstruct @@ -47,11 +56,15 @@ public class HostRequestIntervalRegistry { if (ttlMs < intervalDurationMs) { log.warn("TTL for IntervalRegistry [{}] smaller than interval duration [{}]", ttlMs, intervalDurationMs); } + log.info("Start Host Quota Service with whitelist {}", whiteList); + log.info("Start Host Quota Service with blacklist {}", blackList); } public long tick(String clientHostId) { - if ("localhost".equals(clientHostId) || "127.0.0.1".equals(clientHostId)) { + if (whiteList.contains(clientHostId)) { return 0; + } else if (blackList.contains(clientHostId)) { + return Long.MAX_VALUE; } IntervalCount intervalCount = hostCounts.computeIfAbsent(clientHostId, s -> new IntervalCount(intervalDurationMs)); return intervalCount.resetIfExpiredAndTick(); diff --git a/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java b/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java index 91797d4422..3f660fccb3 100644 --- a/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java +++ b/common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java @@ -80,7 +80,7 @@ public class IntervalRegistryLogger { private void log(Map top) { StringBuilder builder = new StringBuilder("Quota Statistic : "); for (Map.Entry host : top.entrySet()) { - builder.append(host.getKey()).append(" : ").append(host.getValue()); + builder.append(host.getKey()).append(" : ").append(host.getValue()).append(" ; "); } log.info(builder.toString()); diff --git a/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java b/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java index 475fcccead..b8c9284fda 100644 --- a/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java +++ b/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java @@ -44,9 +44,9 @@ public class HostRequestsQuotaServiceTest { } @Test - public void hostQuotaValidated() { + public void quotaExceededIfRequestCountBiggerThanAllowed() { when(requestRegistry.tick("key")).thenReturn(10L); - when(requestsPolicy.isValid(10L)).thenReturn(true); + when(requestsPolicy.isValid(10L)).thenReturn(false); assertTrue(quotaService.isQuotaExceeded("key")); @@ -55,6 +55,18 @@ public class HostRequestsQuotaServiceTest { verifyNoMoreInteractions(requestRegistry, requestsPolicy); } + @Test + public void quotaNotExceededIfRequestCountLessThanAllowed() { + when(requestRegistry.tick("key")).thenReturn(10L); + when(requestsPolicy.isValid(10L)).thenReturn(true); + + assertFalse(quotaService.isQuotaExceeded("key")); + + verify(requestRegistry).tick("key"); + verify(requestsPolicy).isValid(10L); + verifyNoMoreInteractions(requestRegistry, requestsPolicy); + } + @Test public void serviceCanBeDisabled() { quotaService = new HostRequestsQuotaService(requestRegistry, requestsPolicy, registryCleaner, registryLogger, false); diff --git a/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java b/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java index cb21ccebd5..ff1e525f91 100644 --- a/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java +++ b/common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.common.transport.quota.inmemory; +import com.google.common.collect.Sets; import org.junit.Before; import org.junit.Test; +import java.util.Collections; + import static org.junit.Assert.assertEquals; /** @@ -30,7 +33,7 @@ public class HostRequestIntervalRegistryTest { @Before public void init() { - registry = new HostRequestIntervalRegistry(10000L, 100L); + registry = new HostRequestIntervalRegistry(10000L, 100L,"g1,g2", "b1"); } @Test @@ -54,4 +57,29 @@ public class HostRequestIntervalRegistryTest { assertEquals(1L, registry.tick("aaa")); assertEquals(2L, registry.tick("bbb")); } + + @Test + public void domainFromWhitelistNotCounted(){ + assertEquals(0L, registry.tick("g1")); + assertEquals(0L, registry.tick("g1")); + assertEquals(0L, registry.tick("g2")); + } + + @Test + public void domainFromBlackListReturnMaxValue(){ + assertEquals(Long.MAX_VALUE, registry.tick("b1")); + assertEquals(Long.MAX_VALUE, registry.tick("b1")); + } + + @Test + public void emptyWhitelistParsedOk(){ + registry = new HostRequestIntervalRegistry(10000L, 100L,"", "b1"); + assertEquals(1L, registry.tick("aaa")); + } + + @Test + public void emptyBlacklistParsedOk(){ + registry = new HostRequestIntervalRegistry(10000L, 100L,"", ""); + assertEquals(1L, registry.tick("aaa")); + } } \ No newline at end of file diff --git a/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 2ffbf05f6e..958c4a79c8 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -75,7 +75,6 @@ public class CoapTransportResource extends CoapResource { @Override public void handleGET(CoapExchange exchange) { if(quotaService.isQuotaExceeded(exchange.getSourceAddress().getHostAddress())) { - log.warn("Missing feature type parameter"); log.warn("COAP Quota exceeded for [{}:{}] . Disconnect", exchange.getSourceAddress().getHostAddress(), exchange.getSourcePort()); exchange.respond(ResponseCode.BAD_REQUEST); return;