Browse Source

Merged with main branch and fixed tests

pull/804/head
Andrew Shvayka 8 years ago
parent
commit
02f24ca672
  1. 9
      application/src/main/java/org/thingsboard/server/service/queue/DefaultMsgQueueService.java
  2. 25
      application/src/main/resources/thingsboard.yml
  3. 21
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/AbstractQuotaService.java
  4. 30
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/RequestLimitPolicy.java
  5. 29
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostIntervalRegistryCleaner.java
  6. 52
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostIntervalRegistryLogger.java
  7. 37
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestIntervalRegistry.java
  8. 13
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestLimitPolicy.java
  9. 37
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestsQuotaService.java
  10. 10
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryCleaner.java
  11. 24
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java
  12. 25
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/KeyBasedIntervalRegistry.java
  13. 29
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantIntervalRegistryCleaner.java
  14. 52
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantIntervalRegistryLogger.java
  15. 31
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantMsgsIntervalRegistry.java
  16. 30
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantQuotaService.java
  17. 28
      common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantRequestLimitPolicy.java
  18. 1
      common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestLimitPolicyTest.java
  19. 8
      common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java
  20. 4
      common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java
  21. 4
      common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLoggerTest.java
  22. 2
      dao/src/main/java/org/thingsboard/server/dao/queue/db/MsgAck.java
  23. 3
      dao/src/main/java/org/thingsboard/server/dao/queue/db/UnprocessedMsgFilter.java
  24. 2
      dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/CassandraMsgQueue.java
  25. 2
      dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/repository/CassandraAckRepository.java
  26. 2
      dao/src/main/java/org/thingsboard/server/dao/queue/db/repository/AckRepository.java
  27. 97
      dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java
  28. 5
      dao/src/main/java/org/thingsboard/server/dao/relation/RelationService.java
  29. 36
      dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java
  30. 2
      dao/src/test/java/org/thingsboard/server/dao/queue/db/nosql/UnprocessedMsgFilterTest.java
  31. 2
      dao/src/test/java/org/thingsboard/server/dao/queue/db/nosql/repository/CassandraAckRepositoryTest.java
  32. 2
      dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java
  33. 1
      dao/src/test/resources/application-test.properties
  34. 3
      transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java
  35. 6
      transport/coap/src/test/java/org/thingsboard/server/transport/coap/CoapServerTest.java
  36. 3
      transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
  37. 4
      transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java

9
application/src/main/java/org/thingsboard/server/service/queue/DefaultMsgQueueService.java

@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.transport.quota.tenant.TenantQuotaService;
import org.thingsboard.server.dao.queue.MsgQueue; import org.thingsboard.server.dao.queue.MsgQueue;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -48,6 +49,9 @@ public class DefaultMsgQueueService implements MsgQueueService {
@Autowired @Autowired
private MsgQueue msgQueue; private MsgQueue msgQueue;
@Autowired
private TenantQuotaService quotaService;
private ScheduledExecutorService cleanupExecutor; private ScheduledExecutorService cleanupExecutor;
private Map<TenantId, AtomicLong> pendingCountPerTenant = new ConcurrentHashMap<>(); private Map<TenantId, AtomicLong> pendingCountPerTenant = new ConcurrentHashMap<>();
@ -70,6 +74,11 @@ public class DefaultMsgQueueService implements MsgQueueService {
@Override @Override
public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) { public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) {
if(quotaService.isQuotaExceeded(tenantId.getId().toString())) {
log.warn("Tenant TbMsg Quota exceeded for [{}:{}] . Reject", tenantId.getId());
return Futures.immediateFailedFuture(new RuntimeException("Tenant TbMsg Quota exceeded"));
}
AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong()); AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong());
if (pendingMsgCount.incrementAndGet() < queueMaxSize) { if (pendingMsgCount.incrementAndGet() < queueMaxSize) {
return msgQueue.put(tenantId, msg, nodeId, clusterPartition); return msgQueue.put(tenantId, msg, nodeId, clusterPartition);

25
application/src/main/resources/thingsboard.yml

@ -131,9 +131,28 @@ quota:
whitelist: "${QUOTA_HOST_WHITELIST:localhost,127.0.0.1}" whitelist: "${QUOTA_HOST_WHITELIST:localhost,127.0.0.1}"
# Array of blacklist hosts # Array of blacklist hosts
blacklist: "${QUOTA_HOST_BLACKLIST:}" blacklist: "${QUOTA_HOST_BLACKLIST:}"
log: log:
topSize: 10 topSize: 10
intervalMin: 2 intervalMin: 2
rule:
tenant:
# Max allowed number of API requests in interval for single tenant
limit: "${QUOTA_TENANT_LIMIT:100000}"
# Interval duration
intervalMs: "${QUOTA_TENANT_INTERVAL_MS:60000}"
# Maximum silence duration for tenant after which Tenant removed from QuotaService. Must be bigger than intervalMs
ttlMs: "${QUOTA_TENANT_TTL_MS:60000}"
# Interval for scheduled task that cleans expired records. TTL is used for expiring
cleanPeriodMs: "${QUOTA_TENANT_CLEAN_PERIOD_MS:300000}"
# Enable Host API Limits
enabled: "${QUOTA_TENANT_ENABLED:false}"
# Array of whitelist tenants
whitelist: "${QUOTA_TENANT_WHITELIST:}"
# Array of blacklist tenants
blacklist: "${QUOTA_HOST_BLACKLIST:}"
log:
topSize: 10
intervalMin: 2
database: database:
type: "${DATABASE_TYPE:sql}" # cassandra OR sql type: "${DATABASE_TYPE:sql}" # cassandra OR sql

21
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaService.java → common/transport/src/main/java/org/thingsboard/server/common/transport/quota/AbstractQuotaService.java

@ -15,33 +15,24 @@
*/ */
package org.thingsboard.server.common.transport.quota; package org.thingsboard.server.common.transport.quota;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner; import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger; import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
/** public class AbstractQuotaService implements QuotaService {
* @author Vitaliy Paromskiy
* @version 1.0
*/
@Service
@Slf4j
public class HostRequestsQuotaService implements QuotaService {
private final HostRequestIntervalRegistry requestRegistry; private final KeyBasedIntervalRegistry requestRegistry;
private final HostRequestLimitPolicy requestsPolicy; private final RequestLimitPolicy requestsPolicy;
private final IntervalRegistryCleaner registryCleaner; private final IntervalRegistryCleaner registryCleaner;
private final IntervalRegistryLogger registryLogger; private final IntervalRegistryLogger registryLogger;
private final boolean enabled; private final boolean enabled;
public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy, public AbstractQuotaService(KeyBasedIntervalRegistry requestRegistry, RequestLimitPolicy requestsPolicy,
IntervalRegistryCleaner registryCleaner, IntervalRegistryLogger registryLogger, IntervalRegistryCleaner registryCleaner, IntervalRegistryLogger registryLogger,
@Value("${quota.host.enabled}") boolean enabled) { boolean enabled) {
this.requestRegistry = requestRegistry; this.requestRegistry = requestRegistry;
this.requestsPolicy = requestsPolicy; this.requestsPolicy = requestsPolicy;
this.registryCleaner = registryCleaner; this.registryCleaner = registryCleaner;

30
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/RequestLimitPolicy.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota;
public abstract class RequestLimitPolicy {
private final long limit;
public RequestLimitPolicy(long limit) {
this.limit = limit;
}
public boolean isValid(long currentValue) {
return currentValue <= limit;
}
}

29
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostIntervalRegistryCleaner.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.host;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
@Component
public class HostIntervalRegistryCleaner extends IntervalRegistryCleaner {
public HostIntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry,
@Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) {
super(intervalRegistry, cleanPeriodMs);
}
}

52
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostIntervalRegistryLogger.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.host;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@Component
@Slf4j
public class HostIntervalRegistryLogger extends IntervalRegistryLogger {
private final long logIntervalMin;
public HostIntervalRegistryLogger(@Value("${quota.host.log.topSize}") int topSize,
@Value("${quota.host.log.intervalMin}") long logIntervalMin,
HostRequestIntervalRegistry intervalRegistry) {
super(topSize, logIntervalMin, intervalRegistry);
this.logIntervalMin = logIntervalMin;
}
protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) {
long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
StringBuilder builder = new StringBuilder("Quota Statistic : ");
builder.append("uniqHosts : ").append(uniqHosts).append("; ");
builder.append("requestsCount : ").append(requestsCount).append("; ");
builder.append("RPS : ").append(rps).append(" ");
builder.append("top -> ");
for (Map.Entry<String, Long> host : top.entrySet()) {
builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
}
log.info(builder.toString());
}
}

37
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestIntervalRegistry.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.host;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
/**
* @author Vitaliy Paromskiy
* @version 1.0
*/
@Component
@Slf4j
public class HostRequestIntervalRegistry extends KeyBasedIntervalRegistry {
public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs,
@Value("${quota.host.ttlMs}") long ttlMs,
@Value("${quota.host.whitelist}") String whiteList,
@Value("${quota.host.blacklist}") String blackList) {
super(intervalDurationMs, ttlMs, whiteList, blackList, "host");
}
}

13
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/HostRequestLimitPolicy.java → common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestLimitPolicy.java

@ -13,26 +13,21 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.common.transport.quota; package org.thingsboard.server.common.transport.quota.host;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.RequestLimitPolicy;
/** /**
* @author Vitaliy Paromskiy * @author Vitaliy Paromskiy
* @version 1.0 * @version 1.0
*/ */
@Component @Component
public class HostRequestLimitPolicy { public class HostRequestLimitPolicy extends RequestLimitPolicy {
private final long limit;
public HostRequestLimitPolicy(@Value("${quota.host.limit}") long limit) { public HostRequestLimitPolicy(@Value("${quota.host.limit}") long limit) {
this.limit = limit; super(limit);
}
public boolean isValid(long currentValue) {
return currentValue <= limit;
} }
} }

37
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/host/HostRequestsQuotaService.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.host;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.transport.quota.AbstractQuotaService;
/**
* @author Vitaliy Paromskiy
* @version 1.0
*/
@Service
@Slf4j
public class HostRequestsQuotaService extends AbstractQuotaService {
public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy,
HostIntervalRegistryCleaner registryCleaner, HostIntervalRegistryLogger registryLogger,
@Value("${quota.host.enabled}") boolean enabled) {
super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled);
}
}

10
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryCleaner.java

@ -16,10 +16,7 @@
package org.thingsboard.server.common.transport.quota.inmemory; package org.thingsboard.server.common.transport.quota.inmemory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -28,15 +25,14 @@ import java.util.concurrent.TimeUnit;
* @author Vitaliy Paromskiy * @author Vitaliy Paromskiy
* @version 1.0 * @version 1.0
*/ */
@Component
@Slf4j @Slf4j
public class IntervalRegistryCleaner { public abstract class IntervalRegistryCleaner {
private final HostRequestIntervalRegistry intervalRegistry; private final KeyBasedIntervalRegistry intervalRegistry;
private final long cleanPeriodMs; private final long cleanPeriodMs;
private ScheduledExecutorService executor; private ScheduledExecutorService executor;
public IntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry, @Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) { public IntervalRegistryCleaner(KeyBasedIntervalRegistry intervalRegistry, long cleanPeriodMs) {
this.intervalRegistry = intervalRegistry; this.intervalRegistry = intervalRegistry;
this.cleanPeriodMs = cleanPeriodMs; this.cleanPeriodMs = cleanPeriodMs;
} }

24
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLogger.java

@ -17,8 +17,6 @@ package org.thingsboard.server.common.transport.quota.inmemory;
import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.MinMaxPriorityQueue;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Comparator; import java.util.Comparator;
import java.util.Map; import java.util.Map;
@ -32,17 +30,15 @@ import java.util.stream.Collectors;
* @author Vitaliy Paromskiy * @author Vitaliy Paromskiy
* @version 1.0 * @version 1.0
*/ */
@Component
@Slf4j @Slf4j
public class IntervalRegistryLogger { public abstract class IntervalRegistryLogger {
private final int topSize; private final int topSize;
private final HostRequestIntervalRegistry intervalRegistry; private final KeyBasedIntervalRegistry intervalRegistry;
private final long logIntervalMin; private final long logIntervalMin;
private ScheduledExecutorService executor; private ScheduledExecutorService executor;
public IntervalRegistryLogger(@Value("${quota.log.topSize}") int topSize, @Value("${quota.log.intervalMin}") long logIntervalMin, public IntervalRegistryLogger(int topSize, long logIntervalMin, KeyBasedIntervalRegistry intervalRegistry) {
HostRequestIntervalRegistry intervalRegistry) {
this.topSize = topSize; this.topSize = topSize;
this.logIntervalMin = logIntervalMin; this.logIntervalMin = logIntervalMin;
this.intervalRegistry = intervalRegistry; this.intervalRegistry = intervalRegistry;
@ -79,17 +75,5 @@ public class IntervalRegistryLogger {
return topQueue.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return topQueue.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
} }
private void log(Map<String, Long> top, int uniqHosts, long requestsCount) { protected abstract void log(Map<String, Long> top, int uniqHosts, long requestsCount);
long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
StringBuilder builder = new StringBuilder("Quota Statistic : ");
builder.append("uniqHosts : ").append(uniqHosts).append("; ");
builder.append("requestsCount : ").append(requestsCount).append("; ");
builder.append("RPS : ").append(rps).append(" ");
builder.append("top -> ");
for (Map.Entry<String, Long> host : top.entrySet()) {
builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
}
log.info(builder.toString());
}
} }

25
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistry.java → common/transport/src/main/java/org/thingsboard/server/common/transport/quota/inmemory/KeyBasedIntervalRegistry.java

@ -18,22 +18,14 @@ package org.thingsboard.server.common.transport.quota.inmemory;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/**
* @author Vitaliy Paromskiy
* @version 1.0
*/
@Component
@Slf4j @Slf4j
public class HostRequestIntervalRegistry { public abstract class KeyBasedIntervalRegistry {
private final Map<String, IntervalCount> hostCounts = new ConcurrentHashMap<>(); private final Map<String, IntervalCount> hostCounts = new ConcurrentHashMap<>();
private final long intervalDurationMs; private final long intervalDurationMs;
@ -41,23 +33,20 @@ public class HostRequestIntervalRegistry {
private final Set<String> whiteList; private final Set<String> whiteList;
private final Set<String> blackList; private final Set<String> blackList;
public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs, public KeyBasedIntervalRegistry(long intervalDurationMs, long ttlMs, String whiteList, String blackList, String name) {
@Value("${quota.host.ttlMs}") long ttlMs,
@Value("${quota.host.whitelist}") String whiteList,
@Value("${quota.host.blacklist}") String blackList) {
this.intervalDurationMs = intervalDurationMs; this.intervalDurationMs = intervalDurationMs;
this.ttlMs = ttlMs; this.ttlMs = ttlMs;
this.whiteList = Sets.newHashSet(StringUtils.split(whiteList, ',')); this.whiteList = Sets.newHashSet(StringUtils.split(whiteList, ','));
this.blackList = Sets.newHashSet(StringUtils.split(blackList, ',')); this.blackList = Sets.newHashSet(StringUtils.split(blackList, ','));
validate(name);
} }
@PostConstruct private void validate(String name) {
public void init() {
if (ttlMs < intervalDurationMs) { if (ttlMs < intervalDurationMs) {
log.warn("TTL for IntervalRegistry [{}] smaller than interval duration [{}]", ttlMs, intervalDurationMs); log.warn("TTL for {} IntervalRegistry [{}] smaller than interval duration [{}]", name, ttlMs, intervalDurationMs);
} }
log.info("Start Host Quota Service with whitelist {}", whiteList); log.info("Start {} KeyBasedIntervalRegistry with whitelist {}", name, whiteList);
log.info("Start Host Quota Service with blacklist {}", blackList); log.info("Start {} KeyBasedIntervalRegistry with blacklist {}", name, blackList);
} }
public long tick(String clientHostId) { public long tick(String clientHostId) {

29
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantIntervalRegistryCleaner.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.tenant;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
@Component
public class TenantIntervalRegistryCleaner extends IntervalRegistryCleaner {
public TenantIntervalRegistryCleaner(TenantMsgsIntervalRegistry intervalRegistry,
@Value("${quota.rule.tenant.cleanPeriodMs}") long cleanPeriodMs) {
super(intervalRegistry, cleanPeriodMs);
}
}

52
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantIntervalRegistryLogger.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.tenant;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
public class TenantIntervalRegistryLogger extends IntervalRegistryLogger {
private final long logIntervalMin;
public TenantIntervalRegistryLogger(@Value("${quota.rule.tenant.log.topSize}") int topSize,
@Value("${quota.rule.tenant.log.intervalMin}") long logIntervalMin,
TenantMsgsIntervalRegistry intervalRegistry) {
super(topSize, logIntervalMin, intervalRegistry);
this.logIntervalMin = logIntervalMin;
}
protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) {
long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
StringBuilder builder = new StringBuilder("Tenant Quota Statistic : ");
builder.append("uniqTenants : ").append(uniqHosts).append("; ");
builder.append("requestsCount : ").append(requestsCount).append("; ");
builder.append("RPS : ").append(rps).append(" ");
builder.append("top -> ");
for (Map.Entry<String, Long> host : top.entrySet()) {
builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
}
log.info(builder.toString());
}
}

31
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantMsgsIntervalRegistry.java

@ -0,0 +1,31 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.tenant;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
@Component
public class TenantMsgsIntervalRegistry extends KeyBasedIntervalRegistry {
public TenantMsgsIntervalRegistry(@Value("${quota.rule.tenant.intervalMs}") long intervalDurationMs,
@Value("${quota.rule.tenant.ttlMs}") long ttlMs,
@Value("${quota.rule.tenant.whitelist}") String whiteList,
@Value("${quota.rule.tenant.blacklist}") String blackList) {
super(intervalDurationMs, ttlMs, whiteList, blackList, "Rule Tenant");
}
}

30
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantQuotaService.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.tenant;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.AbstractQuotaService;
@Component
public class TenantQuotaService extends AbstractQuotaService {
public TenantQuotaService(TenantMsgsIntervalRegistry requestRegistry, TenantRequestLimitPolicy requestsPolicy,
TenantIntervalRegistryCleaner registryCleaner, TenantIntervalRegistryLogger registryLogger,
@Value("${quota.rule.tenant.enabled}") boolean enabled) {
super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled);
}
}

28
common/transport/src/main/java/org/thingsboard/server/common/transport/quota/tenant/TenantRequestLimitPolicy.java

@ -0,0 +1,28 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.quota.tenant;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.quota.RequestLimitPolicy;
@Component
public class TenantRequestLimitPolicy extends RequestLimitPolicy {
public TenantRequestLimitPolicy(@Value("${quota.rule.tenant.limit}") long limit) {
super(limit);
}
}

1
common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestLimitPolicyTest.java

@ -16,6 +16,7 @@
package org.thingsboard.server.common.transport.quota; package org.thingsboard.server.common.transport.quota;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.server.common.transport.quota.host.HostRequestLimitPolicy;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;

8
common/transport/src/test/java/org/thingsboard/server/common/transport/quota/HostRequestsQuotaServiceTest.java

@ -17,9 +17,7 @@ package org.thingsboard.server.common.transport.quota;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry; import org.thingsboard.server.common.transport.quota.host.*;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -35,8 +33,8 @@ public class HostRequestsQuotaServiceTest {
private HostRequestIntervalRegistry requestRegistry = mock(HostRequestIntervalRegistry.class); private HostRequestIntervalRegistry requestRegistry = mock(HostRequestIntervalRegistry.class);
private HostRequestLimitPolicy requestsPolicy = mock(HostRequestLimitPolicy.class); private HostRequestLimitPolicy requestsPolicy = mock(HostRequestLimitPolicy.class);
private IntervalRegistryCleaner registryCleaner = mock(IntervalRegistryCleaner.class); private HostIntervalRegistryCleaner registryCleaner = mock(HostIntervalRegistryCleaner.class);
private IntervalRegistryLogger registryLogger = mock(IntervalRegistryLogger.class); private HostIntervalRegistryLogger registryLogger = mock(HostIntervalRegistryLogger.class);
@Before @Before
public void init() { public void init() {

4
common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/HostRequestIntervalRegistryTest.java

@ -15,11 +15,9 @@
*/ */
package org.thingsboard.server.common.transport.quota.inmemory; package org.thingsboard.server.common.transport.quota.inmemory;
import com.google.common.collect.Sets;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry;
import java.util.Collections;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;

4
common/transport/src/test/java/org/thingsboard/server/common/transport/quota/inmemory/IntervalRegistryLoggerTest.java

@ -18,6 +18,8 @@ package org.thingsboard.server.common.transport.quota.inmemory;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.server.common.transport.quota.host.HostIntervalRegistryLogger;
import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
@ -37,7 +39,7 @@ public class IntervalRegistryLoggerTest {
@Before @Before
public void init() { public void init() {
logger = new IntervalRegistryLogger(3, 10, requestRegistry); logger = new HostIntervalRegistryLogger(3, 10, requestRegistry);
} }
@Test @Test

2
dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/MsgAck.java → dao/src/main/java/org/thingsboard/server/dao/queue/db/MsgAck.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.dao.queue.db.nosql; package org.thingsboard.server.dao.queue.db;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;

3
dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/UnprocessedMsgFilter.java → dao/src/main/java/org/thingsboard/server/dao/queue/db/UnprocessedMsgFilter.java

@ -13,10 +13,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.dao.queue.db.nosql; package org.thingsboard.server.dao.queue.db;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.dao.queue.db.MsgAck;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;

2
dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/CassandraMsgQueue.java

@ -26,6 +26,8 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.dao.queue.MsgQueue; import org.thingsboard.server.dao.queue.MsgQueue;
import org.thingsboard.server.dao.queue.db.MsgAck;
import org.thingsboard.server.dao.queue.db.UnprocessedMsgFilter;
import org.thingsboard.server.dao.queue.db.repository.AckRepository; import org.thingsboard.server.dao.queue.db.repository.AckRepository;
import org.thingsboard.server.dao.queue.db.repository.MsgRepository; import org.thingsboard.server.dao.queue.db.repository.MsgRepository;
import org.thingsboard.server.dao.util.NoSqlDao; import org.thingsboard.server.dao.util.NoSqlDao;

2
dao/src/main/java/org/thingsboard/server/dao/queue/db/nosql/repository/CassandraAckRepository.java

@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.dao.nosql.CassandraAbstractDao; import org.thingsboard.server.dao.nosql.CassandraAbstractDao;
import org.thingsboard.server.dao.queue.db.nosql.MsgAck; import org.thingsboard.server.dao.queue.db.MsgAck;
import org.thingsboard.server.dao.queue.db.repository.AckRepository; import org.thingsboard.server.dao.queue.db.repository.AckRepository;
import org.thingsboard.server.dao.util.NoSqlDao; import org.thingsboard.server.dao.util.NoSqlDao;

2
dao/src/main/java/org/thingsboard/server/dao/queue/db/repository/AckRepository.java

@ -16,7 +16,7 @@
package org.thingsboard.server.dao.queue.db.repository; package org.thingsboard.server.dao.queue.db.repository;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.dao.queue.db.nosql.MsgAck; import org.thingsboard.server.dao.queue.db.MsgAck;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;

97
dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.relation; package org.thingsboard.server.dao.relation;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -176,97 +177,65 @@ public class BaseRelationService implements RelationService {
} }
@Override @Override
public boolean deleteEntityRelations(EntityId entity) { public void deleteEntityRelations(EntityId entityId) {
Cache cache = cacheManager.getCache(RELATIONS_CACHE);
log.trace("Executing deleteEntityRelations [{}]", entity);
validate(entity);
List<ListenableFuture<List<EntityRelation>>> inboundRelationsList = new ArrayList<>();
for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
inboundRelationsList.add(relationDao.findAllByTo(entity, typeGroup));
}
ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList);
ListenableFuture<List<Boolean>> inboundDeletions = Futures.transform(inboundRelations, relations ->
getBooleans(relations, cache, true));
ListenableFuture<Boolean> inboundFuture = Futures.transform(inboundDeletions, getListToBooleanFunction());
boolean inboundDeleteResult = false;
try { try {
inboundDeleteResult = inboundFuture.get(); deleteEntityRelationsAsync(entityId).get();
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
log.error("Error deleting entity inbound relations", e); throw new RuntimeException(e);
}
List<ListenableFuture<List<EntityRelation>>> outboundRelationsList = new ArrayList<>();
for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
outboundRelationsList.add(relationDao.findAllByFrom(entity, typeGroup));
}
ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
Futures.transform(outboundRelations, relations -> getBooleans(relations, cache, false));
boolean outboundDeleteResult = relationDao.deleteOutboundRelations(entity);
return inboundDeleteResult && outboundDeleteResult;
}
private List<Boolean> getBooleans(List<List<EntityRelation>> relations, Cache cache, boolean isRemove) {
List<Boolean> results = new ArrayList<>();
for (List<EntityRelation> relationList : relations) {
relationList.forEach(relation -> checkFromDeleteSync(cache, results, relation, isRemove));
} }
return results;
}
private void checkFromDeleteSync(Cache cache, List<Boolean> results, EntityRelation relation, boolean isRemove) {
if (isRemove) {
results.add(relationDao.deleteRelation(relation));
}
cacheEviction(relation, cache);
} }
@Override @Override
public ListenableFuture<Boolean> deleteEntityRelationsAsync(EntityId entity) { public ListenableFuture<Void> deleteEntityRelationsAsync(EntityId entityId) {
Cache cache = cacheManager.getCache(RELATIONS_CACHE); Cache cache = cacheManager.getCache(RELATIONS_CACHE);
log.trace("Executing deleteEntityRelationsAsync [{}]", entity); log.trace("Executing deleteEntityRelationsAsync [{}]", entityId);
validate(entity); validate(entityId);
List<ListenableFuture<List<EntityRelation>>> inboundRelationsList = new ArrayList<>(); List<ListenableFuture<List<EntityRelation>>> inboundRelationsList = new ArrayList<>();
for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
inboundRelationsList.add(relationDao.findAllByTo(entity, typeGroup)); inboundRelationsList.add(relationDao.findAllByTo(entityId, typeGroup));
} }
ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList); ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList);
List<ListenableFuture<List<EntityRelation>>> outboundRelationsList = new ArrayList<>();
for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
outboundRelationsList.add(relationDao.findAllByFrom(entityId, typeGroup));
}
ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
ListenableFuture<List<Boolean>> inboundDeletions = Futures.transformAsync(inboundRelations, ListenableFuture<List<Boolean>> inboundDeletions = Futures.transformAsync(inboundRelations,
relations -> { relations -> {
List<ListenableFuture<Boolean>> results = getListenableFutures(relations, cache, true); List<ListenableFuture<Boolean>> results = deleteRelationGroupsAsync(relations, cache, true);
return Futures.allAsList(results); return Futures.allAsList(results);
}); });
ListenableFuture<Boolean> inboundFuture = Futures.transform(inboundDeletions, getListToBooleanFunction()); ListenableFuture<List<Boolean>> outboundDeletions = Futures.transformAsync(outboundRelations,
relations -> {
List<ListenableFuture<Boolean>> results = deleteRelationGroupsAsync(relations, cache, false);
return Futures.allAsList(results);
});
List<ListenableFuture<List<EntityRelation>>> outboundRelationsList = new ArrayList<>(); ListenableFuture<List<List<Boolean>>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions);
for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
outboundRelationsList.add(relationDao.findAllByFrom(entity, typeGroup));
}
ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
Futures.transformAsync(outboundRelations, relations -> {
List<ListenableFuture<Boolean>> results = getListenableFutures(relations, cache, false);
return Futures.allAsList(results);
});
ListenableFuture<Boolean> outboundFuture = relationDao.deleteOutboundRelationsAsync(entity); return Futures.transform(Futures.transformAsync(deletionsFuture, (deletions) -> relationDao.deleteOutboundRelationsAsync(entityId)), result -> null);
return Futures.transform(Futures.allAsList(Arrays.asList(inboundFuture, outboundFuture)), getListToBooleanFunction());
} }
private List<ListenableFuture<Boolean>> getListenableFutures(List<List<EntityRelation>> relations, Cache cache, boolean isRemove) { private List<ListenableFuture<Boolean>> deleteRelationGroupsAsync(List<List<EntityRelation>> relations, Cache cache, boolean deleteFromDb) {
List<ListenableFuture<Boolean>> results = new ArrayList<>(); List<ListenableFuture<Boolean>> results = new ArrayList<>();
for (List<EntityRelation> relationList : relations) { for (List<EntityRelation> relationList : relations) {
relationList.forEach(relation -> checkFromDeleteAsync(cache, results, relation, isRemove)); relationList.forEach(relation -> results.add(deleteAsync(cache, relation, deleteFromDb)));
} }
return results; return results;
} }
private void checkFromDeleteAsync(Cache cache, List<ListenableFuture<Boolean>> results, EntityRelation relation, boolean isRemove) { private ListenableFuture<Boolean> deleteAsync(Cache cache, EntityRelation relation, boolean deleteFromDb) {
if (isRemove) {
results.add(relationDao.deleteRelationAsync(relation));
}
cacheEviction(relation, cache); cacheEviction(relation, cache);
if (deleteFromDb) {
return relationDao.deleteRelationAsync(relation);
} else {
return Futures.immediateFuture(false);
}
} }
private void cacheEviction(EntityRelation relation, Cache cache) { private void cacheEviction(EntityRelation relation, Cache cache) {

5
dao/src/main/java/org/thingsboard/server/dao/relation/RelationService.java

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery;
import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutionException;
/** /**
* Created by ashvayka on 27.04.17. * Created by ashvayka on 27.04.17.
@ -47,9 +48,9 @@ public interface RelationService {
ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup);
boolean deleteEntityRelations(EntityId entity); void deleteEntityRelations(EntityId entity);
ListenableFuture<Boolean> deleteEntityRelationsAsync(EntityId entity); ListenableFuture<Void> deleteEntityRelationsAsync(EntityId entity);
List<EntityRelation> findByFrom(EntityId from, RelationTypeGroup typeGroup); List<EntityRelation> findByFrom(EntityId from, RelationTypeGroup typeGroup);

36
dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java

@ -127,39 +127,35 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
@Override @Override
public boolean deleteRelation(EntityRelation relation) { public boolean deleteRelation(EntityRelation relation) {
RelationCompositeKey key = new RelationCompositeKey(relation); RelationCompositeKey key = new RelationCompositeKey(relation);
boolean relationExistsBeforeDelete = relationRepository.exists(key); return deleteRelationIfExists(key);
relationRepository.delete(key);
return relationExistsBeforeDelete;
} }
@Override @Override
public ListenableFuture<Boolean> deleteRelationAsync(EntityRelation relation) { public ListenableFuture<Boolean> deleteRelationAsync(EntityRelation relation) {
RelationCompositeKey key = new RelationCompositeKey(relation); RelationCompositeKey key = new RelationCompositeKey(relation);
return service.submit( return service.submit(
() -> { () -> deleteRelationIfExists(key));
boolean relationExistsBeforeDelete = relationRepository.exists(key);
relationRepository.delete(key);
return relationExistsBeforeDelete;
});
} }
@Override @Override
public boolean deleteRelation(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { public boolean deleteRelation(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) {
RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup);
boolean relationExistsBeforeDelete = relationRepository.exists(key); return deleteRelationIfExists(key);
relationRepository.delete(key);
return relationExistsBeforeDelete;
} }
@Override @Override
public ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { public ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) {
RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup);
return service.submit( return service.submit(
() -> { () -> deleteRelationIfExists(key));
boolean relationExistsBeforeDelete = relationRepository.exists(key); }
relationRepository.delete(key);
return relationExistsBeforeDelete; private boolean deleteRelationIfExists(RelationCompositeKey key) {
}); boolean relationExistsBeforeDelete = relationRepository.exists(key);
if (relationExistsBeforeDelete) {
relationRepository.delete(key);
}
return relationExistsBeforeDelete;
} }
@Override @Override
@ -167,7 +163,9 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
boolean relationExistsBeforeDelete = relationRepository boolean relationExistsBeforeDelete = relationRepository
.findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name()) .findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name())
.size() > 0; .size() > 0;
relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name()); if (relationExistsBeforeDelete) {
relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
}
return relationExistsBeforeDelete; return relationExistsBeforeDelete;
} }
@ -178,7 +176,9 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
boolean relationExistsBeforeDelete = relationRepository boolean relationExistsBeforeDelete = relationRepository
.findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name()) .findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name())
.size() > 0; .size() > 0;
relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name()); if (relationExistsBeforeDelete) {
relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
}
return relationExistsBeforeDelete; return relationExistsBeforeDelete;
}); });
} }

2
dao/src/test/java/org/thingsboard/server/dao/queue/db/nosql/UnprocessedMsgFilterTest.java

@ -18,6 +18,8 @@ package org.thingsboard.server.dao.queue.db.nosql;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.dao.queue.db.MsgAck;
import org.thingsboard.server.dao.queue.db.UnprocessedMsgFilter;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;

2
dao/src/test/java/org/thingsboard/server/dao/queue/db/nosql/repository/CassandraAckRepositoryTest.java

@ -21,9 +21,9 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Test; import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.util.ReflectionTestUtils; import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.server.dao.queue.db.nosql.MsgAck;
import org.thingsboard.server.dao.service.AbstractServiceTest; import org.thingsboard.server.dao.service.AbstractServiceTest;
import org.thingsboard.server.dao.service.DaoNoSqlTest; import org.thingsboard.server.dao.service.DaoNoSqlTest;
import org.thingsboard.server.dao.queue.db.MsgAck;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;

2
dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java

@ -96,7 +96,7 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest {
saveRelation(relationA); saveRelation(relationA);
saveRelation(relationB); saveRelation(relationB);
Assert.assertTrue(relationService.deleteEntityRelationsAsync(childId).get()); Assert.assertNull(relationService.deleteEntityRelationsAsync(childId).get());
Assert.assertFalse(relationService.checkRelation(parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); Assert.assertFalse(relationService.checkRelation(parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get());

1
dao/src/test/resources/application-test.properties

@ -30,4 +30,3 @@ redis.connection.db=0
redis.connection.password= redis.connection.password=
rule.queue.type=memory rule.queue.type=memory
rule.queue.max_size=10000

3
transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java

@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.transport.SessionMsgProcessor; import org.thingsboard.server.common.transport.SessionMsgProcessor;
import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.common.transport.auth.DeviceAuthService;
import org.thingsboard.server.common.transport.quota.QuotaService; import org.thingsboard.server.common.transport.quota.QuotaService;
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -55,7 +56,7 @@ public class CoapTransportService {
private DeviceAuthService authService; private DeviceAuthService authService;
@Autowired(required = false) @Autowired(required = false)
private QuotaService quotaService; private HostRequestsQuotaService quotaService;
@Value("${coap.bind_address}") @Value("${coap.bind_address}")

6
transport/coap/src/test/java/org/thingsboard/server/transport/coap/CoapServerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.session.*;
import org.thingsboard.server.common.transport.SessionMsgProcessor; import org.thingsboard.server.common.transport.SessionMsgProcessor;
import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthResult;
import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.common.transport.auth.DeviceAuthService;
import org.thingsboard.server.common.transport.quota.QuotaService; import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -134,8 +134,8 @@ public class CoapServerTest {
} }
@Bean @Bean
public static QuotaService quotaService() { public static HostRequestsQuotaService quotaService() {
return key -> false; return new HostRequestsQuotaService(null, null, null, null, false);
} }
} }

3
transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java

@ -36,6 +36,7 @@ import org.thingsboard.server.common.transport.SessionMsgProcessor;
import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.common.transport.auth.DeviceAuthService;
import org.thingsboard.server.common.transport.quota.QuotaService; import org.thingsboard.server.common.transport.quota.QuotaService;
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
import org.thingsboard.server.transport.http.session.HttpSessionCtx; import org.thingsboard.server.transport.http.session.HttpSessionCtx;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -61,7 +62,7 @@ public class DeviceApiController {
private DeviceAuthService authService; private DeviceAuthService authService;
@Autowired(required = false) @Autowired(required = false)
private QuotaService quotaService; private HostRequestsQuotaService quotaService;
@RequestMapping(value = "/{deviceToken}/attributes", method = RequestMethod.GET, produces = "application/json") @RequestMapping(value = "/{deviceToken}/attributes", method = RequestMethod.GET, produces = "application/json")
public DeferredResult<ResponseEntity> getDeviceAttributes(@PathVariable("deviceToken") String deviceToken, public DeferredResult<ResponseEntity> getDeviceAttributes(@PathVariable("deviceToken") String deviceToken,

4
transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java

@ -29,7 +29,7 @@ import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.transport.SessionMsgProcessor; import org.thingsboard.server.common.transport.SessionMsgProcessor;
import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.common.transport.auth.DeviceAuthService;
import org.thingsboard.server.common.transport.quota.QuotaService; import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor;
@ -67,7 +67,7 @@ public class MqttTransportService {
private MqttSslHandlerProvider sslHandlerProvider; private MqttSslHandlerProvider sslHandlerProvider;
@Autowired(required = false) @Autowired(required = false)
private QuotaService quotaService; private HostRequestsQuotaService quotaService;
@Value("${mqtt.bind_address}") @Value("${mqtt.bind_address}")
private String host; private String host;

Loading…
Cancel
Save