Browse Source

Merge afeca719f6 into a430c4ddee

pull/15149/merge
Sergii Matviienko 6 days ago
committed by GitHub
parent
commit
9b2806a939
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 25
      application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java
  2. 18
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  3. 33
      application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java
  4. 136
      application/src/test/java/org/thingsboard/server/service/apiusage/TenantApiUsageStateDailyTest.java
  5. 6
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  6. 8
      common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java

25
application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java

@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
public abstract class BaseApiUsageState { public abstract class BaseApiUsageState {
private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>(); private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>(); private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
private final Map<ApiUsageRecordKey, Long> currentDayValues = new ConcurrentHashMap<>();
private final Map<ApiUsageRecordKey, Map<String, Long>> lastGaugesByServiceId = new HashMap<>(); private final Map<ApiUsageRecordKey, Map<String, Long>> lastGaugesByServiceId = new HashMap<>();
private final Map<ApiUsageRecordKey, Long> gaugesReportCycles = new HashMap<>(); private final Map<ApiUsageRecordKey, Long> gaugesReportCycles = new HashMap<>();
@ -48,6 +49,8 @@ public abstract class BaseApiUsageState {
private volatile long nextCycleTs; private volatile long nextCycleTs;
@Getter @Getter
private volatile long currentHourTs; private volatile long currentHourTs;
@Getter
private volatile long currentDayTs;
@Setter @Setter
private long gaugeReportInterval; private long gaugeReportInterval;
@ -57,30 +60,36 @@ public abstract class BaseApiUsageState {
this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
this.currentDayTs = SchedulerUtils.getStartOfCurrentDay();
} }
public StatsCalculationResult calculate(ApiUsageRecordKey key, long value, String serviceId) { public StatsCalculationResult calculate(ApiUsageRecordKey key, long value, String serviceId) {
long currentValue = get(key); long currentValue = get(key);
long currentHourlyValue = getHourly(key); long currentHourlyValue = getHourly(key);
long currentDailyValue = getDaily(key);
StatsCalculationResult result; StatsCalculationResult result;
if (key.isCounter()) { if (key.isCounter()) {
result = StatsCalculationResult.builder() result = StatsCalculationResult.builder()
.newValue(currentValue + value).valueChanged(true) .newValue(currentValue + value).valueChanged(true)
.newHourlyValue(currentHourlyValue + value).hourlyValueChanged(true) .newHourlyValue(currentHourlyValue + value).hourlyValueChanged(true)
.newDailyValue(currentDailyValue + value).dailyValueChanged(true)
.build(); .build();
} else { } else {
Long newGaugeValue = calculateGauge(key, value, serviceId); Long newGaugeValue = calculateGauge(key, value, serviceId);
long newValue = newGaugeValue != null ? newGaugeValue : currentValue; long newValue = newGaugeValue != null ? newGaugeValue : currentValue;
long newHourlyValue = newGaugeValue != null ? Math.max(newGaugeValue, currentHourlyValue) : currentHourlyValue; long newHourlyValue = newGaugeValue != null ? Math.max(newGaugeValue, currentHourlyValue) : currentHourlyValue;
long newDailyValue = newGaugeValue != null ? Math.max(newGaugeValue, currentDailyValue) : currentDailyValue;
result = StatsCalculationResult.builder() result = StatsCalculationResult.builder()
.newValue(newValue).valueChanged(newValue != currentValue || !currentCycleValues.containsKey(key)) .newValue(newValue).valueChanged(newValue != currentValue || !currentCycleValues.containsKey(key))
.newHourlyValue(newHourlyValue).hourlyValueChanged(newHourlyValue != currentHourlyValue || !currentHourValues.containsKey(key)) .newHourlyValue(newHourlyValue).hourlyValueChanged(newHourlyValue != currentHourlyValue || !currentHourValues.containsKey(key))
.newDailyValue(newDailyValue).dailyValueChanged(newDailyValue != currentDailyValue || !currentDayValues.containsKey(key))
.build(); .build();
} }
set(key, result.getNewValue()); set(key, result.getNewValue());
setHourly(key, result.getNewHourlyValue()); setHourly(key, result.getNewHourlyValue());
setDaily(key, result.getNewDailyValue());
return result; return result;
} }
@ -118,6 +127,14 @@ public abstract class BaseApiUsageState {
return currentHourValues.getOrDefault(key, 0L); return currentHourValues.getOrDefault(key, 0L);
} }
public void setDaily(ApiUsageRecordKey key, Long value) {
currentDayValues.put(key, value);
}
public long getDaily(ApiUsageRecordKey key) {
return currentDayValues.getOrDefault(key, 0L);
}
public void setHour(long currentHourTs) { public void setHour(long currentHourTs) {
this.currentHourTs = currentHourTs; this.currentHourTs = currentHourTs;
currentHourValues.clear(); currentHourValues.clear();
@ -125,6 +142,11 @@ public abstract class BaseApiUsageState {
gaugesReportCycles.clear(); gaugesReportCycles.clear();
} }
public void setDay(long currentDayTs) {
this.currentDayTs = currentDayTs;
currentDayValues.clear();
}
public void setCycles(long currentCycleTs, long nextCycleTs) { public void setCycles(long currentCycleTs, long nextCycleTs) {
this.currentCycleTs = currentCycleTs; this.currentCycleTs = currentCycleTs;
this.nextCycleTs = nextCycleTs; this.nextCycleTs = nextCycleTs;
@ -207,6 +229,7 @@ public abstract class BaseApiUsageState {
", currentCycleTs=" + currentCycleTs + ", currentCycleTs=" + currentCycleTs +
", nextCycleTs=" + nextCycleTs + ", nextCycleTs=" + nextCycleTs +
", currentHourTs=" + currentHourTs + ", currentHourTs=" + currentHourTs +
", currentDayTs=" + currentDayTs +
'}'; '}';
} }
@ -217,6 +240,8 @@ public abstract class BaseApiUsageState {
private final boolean valueChanged; private final boolean valueChanged;
private final long newHourlyValue; private final long newHourlyValue;
private final boolean hourlyValueChanged; private final boolean hourlyValueChanged;
private final long newDailyValue;
private final boolean dailyValueChanged;
} }
} }

18
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -92,6 +92,7 @@ import java.util.stream.Collectors;
public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService<EntityId> implements TbApiUsageStateService { public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService<EntityId> implements TbApiUsageStateService {
public static final String HOURLY = "Hourly"; public static final String HOURLY = "Hourly";
public static final String DAILY = "Daily";
private final PartitionService partitionService; private final PartitionService partitionService;
private final TenantService tenantService; private final TenantService tenantService;
@ -199,8 +200,13 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
if (newHourTs != hourTs) { if (newHourTs != hourTs) {
usageState.setHour(newHourTs); usageState.setHour(newHourTs);
} }
long dayTs = usageState.getCurrentDayTs();
long newDayTs = SchedulerUtils.getStartOfCurrentDay();
if (newDayTs != dayTs) {
usageState.setDay(newDayTs);
}
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
log.trace("[{}][{}] Processing usage stats from {} (currentCycleTs={}, currentHourTs={}): {}", tenantId, ownerId, serviceId, ts, newHourTs, values); log.trace("[{}][{}] Processing usage stats from {} (currentCycleTs={}, currentHourTs={}, currentDayTs={}): {}", tenantId, ownerId, serviceId, ts, newHourTs, newDayTs, values);
} }
updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
Set<ApiFeature> apiFeatures = new HashSet<>(); Set<ApiFeature> apiFeatures = new HashSet<>();
@ -223,6 +229,10 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
long newHourlyValue = calculationResult.getNewHourlyValue(); long newHourlyValue = calculationResult.getNewHourlyValue();
updatedEntries.add(new BasicTsKvEntry(newHourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue))); updatedEntries.add(new BasicTsKvEntry(newHourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
} }
if (calculationResult.isDailyValueChanged()) {
long newDailyValue = calculationResult.getNewDailyValue();
updatedEntries.add(new BasicTsKvEntry(newDayTs, new LongDataEntry(recordKey.getApiCountKey() + DAILY, newDailyValue)));
}
if (recordKey.getApiFeature() != null) { if (recordKey.getApiFeature() != null) {
apiFeatures.add(recordKey.getApiFeature()); apiFeatures.add(recordKey.getApiFeature());
} }
@ -511,6 +521,7 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
boolean cycleEntryFound = false; boolean cycleEntryFound = false;
boolean hourlyEntryFound = false; boolean hourlyEntryFound = false;
boolean dailyEntryFound = false;
for (TsKvEntry tsKvEntry : dbValues) { for (TsKvEntry tsKvEntry : dbValues) {
if (tsKvEntry.getKey().equals(key.getApiCountKey())) { if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
cycleEntryFound = true; cycleEntryFound = true;
@ -524,8 +535,11 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
} else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
hourlyEntryFound = true; hourlyEntryFound = true;
state.setHourly(key, tsKvEntry.getTs() == state.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L); state.setHourly(key, tsKvEntry.getTs() == state.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
} else if (tsKvEntry.getKey().equals(key.getApiCountKey() + DAILY)) {
dailyEntryFound = true;
state.setDaily(key, tsKvEntry.getTs() == state.getCurrentDayTs() ? tsKvEntry.getLongValue().get() : 0L);
} }
if (cycleEntryFound && hourlyEntryFound) { if (cycleEntryFound && hourlyEntryFound && dailyEntryFound) {
break; break;
} }
} }

33
application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java

@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import java.util.Arrays; import java.util.Arrays;
@ -32,6 +33,7 @@ import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit;
public class TenantApiUsageState extends BaseApiUsageState { public class TenantApiUsageState extends BaseApiUsageState {
@Getter @Getter
@ -63,21 +65,46 @@ public class TenantApiUsageState extends BaseApiUsageState {
return tenantProfileData.getConfiguration().getWarnThreshold(key); return tenantProfileData.getConfiguration().getWarnThreshold(key);
} }
/**
* Daily quota = monthlyQuota * peakDays / daysInBillingMonth.
* Returns 0 (unlimited) when the monthly quota is unlimited.
*/
public long getDailyThreshold(ApiUsageRecordKey key) {
long monthly = getProfileThreshold(key);
if (monthly == 0) return 0;
int peakDays = ((DefaultTenantProfileConfiguration) tenantProfileData.getConfiguration()).getDailyPeakDays();
long monthMs = getNextCycleTs() - getCurrentCycleTs();
if (monthMs <= 0) return monthly;
return Math.max(1L, monthly * peakDays * TimeUnit.DAYS.toMillis(1) / monthMs);
}
public long getDailyWarnThreshold(ApiUsageRecordKey key) {
long daily = getDailyThreshold(key);
if (daily == 0) return 0;
double warnFraction = ((DefaultTenantProfileConfiguration) tenantProfileData.getConfiguration()).getWarnThreshold();
return (long) (daily * (warnFraction > 0.0 ? warnFraction : 0.8));
}
private Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) { private Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED; ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) { for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
long value = get(recordKey); long value = get(recordKey);
long dailyValue = getDaily(recordKey);
boolean featureEnabled = getProfileFeatureEnabled(recordKey); boolean featureEnabled = getProfileFeatureEnabled(recordKey);
ApiUsageStateValue tmpValue; ApiUsageStateValue tmpValue;
if (featureEnabled) { if (featureEnabled) {
long threshold = getProfileThreshold(recordKey); long threshold = getProfileThreshold(recordKey);
long warnThreshold = getProfileWarnThreshold(recordKey); long warnThreshold = getProfileWarnThreshold(recordKey);
if (threshold == 0 || value == 0 || value < warnThreshold) { long dailyThreshold = getDailyThreshold(recordKey);
long dailyWarnThreshold = getDailyWarnThreshold(recordKey);
if (threshold == 0) {
tmpValue = ApiUsageStateValue.ENABLED; tmpValue = ApiUsageStateValue.ENABLED;
} else if (value < threshold) { } else if (value >= threshold || (dailyThreshold > 0 && dailyValue >= dailyThreshold)) {
tmpValue = ApiUsageStateValue.DISABLED;
} else if (value >= warnThreshold || (dailyThreshold > 0 && dailyValue >= dailyWarnThreshold)) {
tmpValue = ApiUsageStateValue.WARNING; tmpValue = ApiUsageStateValue.WARNING;
} else { } else {
tmpValue = ApiUsageStateValue.DISABLED; tmpValue = ApiUsageStateValue.ENABLED;
} }
} else { } else {
tmpValue = ApiUsageStateValue.DISABLED; tmpValue = ApiUsageStateValue.DISABLED;

136
application/src/test/java/org/thingsboard/server/service/apiusage/TenantApiUsageStateDailyTest.java

@ -0,0 +1,136 @@
/**
* Copyright © 2016-2026 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.msg.tools.SchedulerUtils;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.within;
public class TenantApiUsageStateDailyTest {
private static final long MONTHLY = 10_000_000L;
private static final double WARN_FRACTION = 0.8;
private static final int PEAK_DAYS = 3;
private TenantProfile tenantProfile;
private long currentCycleTs;
private long nextCycleTs;
@Before
public void setUp() {
currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
nextCycleTs = SchedulerUtils.getStartOfNextMonth();
DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder()
.maxTransportMessages(MONTHLY)
.warnThreshold(WARN_FRACTION)
.dailyPeakDays(PEAK_DAYS)
.build();
TenantProfileData profileData = new TenantProfileData();
profileData.setConfiguration(config);
tenantProfile = new TenantProfile();
tenantProfile.setProfileData(profileData);
}
private TenantApiUsageState createState() {
ApiUsageState apiUsageState = new ApiUsageState(new ApiUsageStateId(UUID.randomUUID()));
apiUsageState.setTenantId(TenantId.fromUUID(UUID.randomUUID()));
apiUsageState.setEntityId(TenantId.fromUUID(UUID.randomUUID()));
return new TenantApiUsageState(tenantProfile, apiUsageState);
}
@Test
public void testDailyThresholdIsProportionalToMonthly() {
TenantApiUsageState state = createState();
long daily = state.getDailyThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT);
long monthMs = nextCycleTs - currentCycleTs;
long expected = MONTHLY * PEAK_DAYS * TimeUnit.DAYS.toMillis(1) / monthMs;
assertThat(daily).isEqualTo(expected);
// Roughly 3/30 of monthly for a 30-day month
double ratio = (double) daily / MONTHLY;
assertThat(ratio).isCloseTo((double) PEAK_DAYS / 30, within(0.01));
}
@Test
public void testDailyThresholdIsLessThanMonthly() {
TenantApiUsageState state = createState();
assertThat(state.getDailyThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT))
.isLessThan(MONTHLY);
}
@Test
public void testUnlimitedMonthlyGivesUnlimitedDaily() {
DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder()
.maxTransportMessages(0L)
.build();
TenantProfileData profileData = new TenantProfileData();
profileData.setConfiguration(config);
tenantProfile.setProfileData(profileData);
TenantApiUsageState state = createState();
assertThat(state.getDailyThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT)).isZero();
}
@Test
public void testDailyWarnThresholdAppliesWarnFraction() {
TenantApiUsageState state = createState();
long daily = state.getDailyThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT);
long dailyWarn = state.getDailyWarnThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT);
assertThat((double) dailyWarn / daily).isCloseTo(WARN_FRACTION, within(0.001));
}
@Test
public void testMonthlyThresholdUnchanged() {
TenantApiUsageState state = createState();
assertThat(state.getProfileThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT)).isEqualTo(MONTHLY);
}
@Test
public void testDailyThresholdRespectsPeakDaysConfig() {
// peakDays=1 → daily ≈ 1/30 monthly; peakDays=6 → daily ≈ 6/30 monthly
for (int days : new int[]{1, 3, 6}) {
DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder()
.maxTransportMessages(MONTHLY)
.warnThreshold(WARN_FRACTION)
.dailyPeakDays(days)
.build();
TenantProfileData profileData = new TenantProfileData();
profileData.setConfiguration(config);
tenantProfile.setProfileData(profileData);
TenantApiUsageState state = createState();
double ratio = (double) state.getDailyThreshold(ApiUsageRecordKey.TRANSPORT_MSG_COUNT) / MONTHLY;
assertThat(ratio).isCloseTo((double) days / 30, within(0.01));
}
}
}

6
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -169,6 +169,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
private double warnThreshold; private double warnThreshold;
// Number of "peak days" used to compute the daily quota.
// Daily quota = monthlyQuota * dailyPeakDays / daysInBillingMonth.
// Default of 3 means a tenant may use up to 3× the daily fair-share per day.
@Builder.Default
private int dailyPeakDays = 3;
@Schema(example = "5") @Schema(example = "5")
private long maxCalculatedFieldsPerEntity = 5; private long maxCalculatedFieldsPerEntity = 5;
@Schema(example = "10") @Schema(example = "10")

8
common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java

@ -36,6 +36,14 @@ public class SchedulerUtils {
return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of);
} }
public static long getStartOfCurrentDay() {
return getStartOfCurrentDay(UTC);
}
public static long getStartOfCurrentDay(ZoneId zoneId) {
return LocalDate.now(UTC).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfCurrentHour() { public static long getStartOfCurrentHour() {
return getStartOfCurrentHour(UTC); return getStartOfCurrentHour(UTC);
} }

Loading…
Cancel
Save