tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fixed tests, added error without stacktrace and param to system params

pull/12678/head
IrynaMatveieva 2 years ago
parent
eae72065ee
commit
7911384f37
11 changed files with 125 additions and 116 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 74
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 14
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  3. 11
      application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
  4. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  5. 6
      application/src/main/resources/thingsboard.yml
  6. 55
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java
  7. 10
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java
  8. 7
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java
  9. 60
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java
  10. 1
      common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
  11. 1
      common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java

74
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
View File

@ -43,7 +43,6 @@ import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.cache.limits.RateLimitService;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.LifecycleEvent;
@ -624,6 +623,14 @@ public class ActorSystemContext {
@Getter
private String debugPerTenantLimitsConfiguration;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}")
@Getter
private boolean calculatedFieldsDebugPerTenantEnabled;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
@Getter
private String calculatedFieldsDebugPerTenantLimitsConfiguration;
@Value("${actors.rpc.submit_strategy:BURST}")
@Getter
private String rpcSubmitStrategy;
@ -810,39 +817,42 @@ public class ActorSystemContext {
}
public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map<String, ArgumentEntry> arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) {
if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) {
throw new TbRateLimitsException(EntityType.TENANT);
}
try {
CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder()
.tenantId(tenantId)
.entityId(calculatedFieldId.getId())
.serviceId(getServiceId())
.calculatedFieldId(calculatedFieldId)
.eventEntity(entityId);
if (tbMsgId != null) {
eventBuilder.msgId(tbMsgId);
}
if (tbMsgType != null) {
eventBuilder.msgType(tbMsgType.name());
}
if (arguments != null) {
eventBuilder.arguments(JacksonUtil.toString(
arguments.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg()))
));
}
if (result != null) {
eventBuilder.result(result);
}
if (error != null) {
eventBuilder.error(toString(error));
if (calculatedFieldsDebugPerTenantEnabled) {
if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) {
log.trace("[{}] Calculated field debug event limits exceeded!", tenantId);
throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!");
}
try {
CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder()
.tenantId(tenantId)
.entityId(calculatedFieldId.getId())
.serviceId(getServiceId())
.calculatedFieldId(calculatedFieldId)
.eventEntity(entityId);
if (tbMsgId != null) {
eventBuilder.msgId(tbMsgId);
}
if (tbMsgType != null) {
eventBuilder.msgType(tbMsgType.name());
}
if (arguments != null) {
eventBuilder.arguments(JacksonUtil.toString(
arguments.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg()))
));
}
if (result != null) {
eventBuilder.result(result);
}
if (error != null) {
eventBuilder.error(error.getMessage());
}
ListenableFuture<Void> future = eventService.saveAsync(eventBuilder.build());
Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} catch (IllegalArgumentException ex) {
log.warn("Failed to persist calculated field debug message", ex);
ListenableFuture<Void> future = eventService.saveAsync(eventBuilder.build());
Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} catch (IllegalArgumentException ex) {
log.warn("Failed to persist calculated field debug message", ex);
}
}
}

14
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
View File

@ -262,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
callback.onSuccess();
} else {
var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService());
try {
newCfCtx.init();
} catch (Exception e) {
if (DebugModeUtil.isDebugAllAvailable(newCf)) {
systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e);
}
}
calculatedFields.put(newCf.getId(), newCfCtx);
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId());
List<CalculatedFieldCtx> newCfList = new ArrayList<>(oldCfList.size());
@ -286,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx);
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) {
try {
newCfCtx.init();
} catch (Exception e) {
if (DebugModeUtil.isDebugAllAvailable(newCf)) {
systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e);
}
}
initCf(newCfCtx, callback, stateChanges);
} else {
callback.onSuccess();

11
application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
View File

@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.settings.UserSettings;
import org.thingsboard.server.common.data.settings.UserSettingsType;
@ -80,6 +80,12 @@ public class SystemInfoController extends BaseController {
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
private String ruleChainDebugPerTenantLimitsConfiguration;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}")
private boolean calculatedFieldDebugPerTenantLimitsEnabled;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
private String calculatedFieldDebugPerTenantLimitsConfiguration;
@Autowired(required = false)
private BuildProperties buildProperties;
@ -155,6 +161,9 @@ public class SystemInfoController extends BaseController {
if (ruleChainDebugPerTenantLimitsEnabled) {
systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration);
}
if (calculatedFieldDebugPerTenantLimitsEnabled) {
systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration);
}
}
systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID))
.map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage)

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
View File

@ -43,9 +43,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
private TreeMap<Long, Double> tsRecords = new TreeMap<>();
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries, int limit, long timeWindow) {
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry));
this.limit = limit;
this.timeWindow = timeWindow;
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry));
}
public TsRollingArgumentEntry(TreeMap<Long, Double> tsRecords, int limit, long timeWindow) {

6
application/src/main/resources/thingsboard.yml
View File

@ -510,6 +510,12 @@ actors:
js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}"
# Actors statistic persistence frequency in milliseconds
persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}"
calculated_fields:
debug_mode_rate_limits_per_tenant:
# Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant
enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}"
# The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour
configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}"
debug:
settings:

55
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java
View File

@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.service.cf.CalculatedFieldResult;
@ -57,7 +57,7 @@ public class ScriptCalculatedFieldStateTest {
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb"));
private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76"));
private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L);
private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L);
private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry();
private final long ts = System.currentTimeMillis();
@ -127,52 +127,7 @@ public class ScriptCalculatedFieldStateTest {
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L));
}
@Test
void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException {
TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 40000, 4.0);// will not be used for calculation
values.put(ts - 45000, 2.0);// will not be used for calculation
values.put(ts - 20, 0.0);
argumentEntry.setTsRecords(values);
state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry));
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L));
}
@Test
void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException {
TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 1000.0);// will not be used
values.put(ts - 18, 0.0);
values.put(ts - 16, 0.0);
values.put(ts - 14, 0.0);
values.put(ts - 12, 0.0);
values.put(ts - 10, 0.0);
argumentEntry.setTsRecords(values);
state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry));
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L));
assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0));
}
@Test
@ -189,7 +144,7 @@ public class ScriptCalculatedFieldStateTest {
@Test
void testIsReadyWhenEmptyEntryPresents() {
// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry));
state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry));
assertThat(state.isReady()).isFalse();
}
@ -235,7 +190,7 @@ public class ScriptCalculatedFieldStateTest {
config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2));
config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;");
config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}");
Output output = new Output();
output.setType(OutputType.ATTRIBUTES);

10
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java
View File

@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest {
"key2", key2ArgEntry
));
// Map<String, ArgumentEntry> newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY);
// assertThatThrownBy(() -> state.updateState(newArgs))
// .isInstanceOf(IllegalArgumentException.class)
// .hasMessage("Rolling argument entry is not supported for simple calculated fields.");
Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L));
assertThatThrownBy(() -> state.updateState(newArgs))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Rolling argument entry is not supported for simple calculated fields.");
}
@Test
@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest {
"key1", key1ArgEntry,
"key2", key2ArgEntry
));
// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY);
state.getArguments().put("key3", new SingleValueArgumentEntry());
assertThat(state.isReady()).isFalse();
}

7
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java
View File

@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class SingleValueArgumentEntryTest {
@ -39,9 +40,9 @@ public class SingleValueArgumentEntryTest {
@Test
void testUpdateEntryWhenRollingEntryPassed() {
// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY))
// .isInstanceOf(IllegalArgumentException.class)
// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING);
assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING);
}
@Test

60
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java
View File

@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest {
values.put(ts - 30, 12.0);
values.put(ts - 20, 17.0);
// entry = new TsRollingArgumentEntry(values);
entry = new TsRollingArgumentEntry(5, 30000L, values);
}
@Test
@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest {
assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0);
}
@Test
void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() {
SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L);
assertThat(entry.updateEntry(newEntry)).isFalse();
}
@Test
void testUpdateEntryWhenRollingEntryPassed() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 16.0);
values.put(ts - 10, 7.0);
values.put(ts - 5, 1.0);
newEntry.setTsRecords(values);
@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest {
assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(5);
assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 40, new DoubleDataEntry("key", 10.0),
ts - 30, new DoubleDataEntry("key", 12.0),
ts - 20, new DoubleDataEntry("key", 17.0),
ts - 10, new DoubleDataEntry("key", 7.0),
ts - 5, new DoubleDataEntry("key", 1.0)
ts - 40, 10.0,
ts - 30, 12.0,
ts - 20, 17.0,
ts - 10, 7.0,
ts - 5, 1.0
));
}
@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest {
assertThatThrownBy(() -> entry.updateEntry(newEntry))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values.");
.hasMessage("Time series rolling arguments supports only numeric values.");
}
@Test
void testUpdateEntryWhenOldTelemetry() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 40000, 4.0);// will not be used for calculation
values.put(ts - 45000, 2.0);// will not be used for calculation
values.put(ts - 5, 0.0);
newEntry.setTsRecords(values);
entry = new TsRollingArgumentEntry(3, 30000L);
assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(1);
assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 5, 0.0
));
}
@Test
void testPerformCalculationWhenArgumentsMoreThanLimit() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 1000.0);// will not be used
values.put(ts - 18, 0.0);
values.put(ts - 16, 0.0);
values.put(ts - 14, 0.0);
newEntry.setTsRecords(values);
entry = new TsRollingArgumentEntry(3, 30000L);
assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(3);
assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 18, 0.0,
ts - 16, 0.0,
ts - 14, 0.0
));
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
View File

@ -34,4 +34,5 @@ public class SystemParams {
boolean mobileQrEnabled;
int maxDebugModeDurationMinutes;
String ruleChainDebugPerTenantLimitsConfiguration;
String calculatedFieldDebugPerTenantLimitsConfiguration;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java
View File

@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event {
return eventInfo;
}
}

Write Preview
Loading…
Cancel
Save