From 0295fefb023e99eec685a9a76a0ff8770558d2ea Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Feb 2025 12:02:39 +0200 Subject: [PATCH 1/8] fixed limits --- ...alculatedFieldManagerMessageProcessor.java | 6 +- .../ctx/state/BaseCalculatedFieldState.java | 16 ++-- .../cf/ctx/state/CalculatedFieldState.java | 2 + .../cf/ctx/state/RocksDBStateService.java | 74 +++++++------------ .../DefaultTenantProfileConfiguration.java | 6 +- common/proto/src/main/proto/queue.proto | 22 ++---- .../CalculatedFieldDataValidator.java | 14 +--- 7 files changed, 49 insertions(+), 91 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index b2ff883236..ec86e640d4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -119,7 +119,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { - if (calculatedFields.containsKey(msg.getId().cfId())) { + var cfId = msg.getId().cfId(); + var calculatedField = calculatedFields.get(cfId); + + if (calculatedField != null) { + msg.getState().setRequiredArguments(calculatedField.getArgNames()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index f5f681b505..21105a44cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.NoArgsConstructor; +import lombok.Data; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -@NoArgsConstructor +@Data public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; @@ -32,14 +33,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { this.arguments = new HashMap<>(); } - @Override - public Map getArguments() { - return arguments; - } - - @Override - public List getRequiredArguments() { - return requiredArguments; + public BaseCalculatedFieldState() { + this.requiredArguments = new ArrayList<>(); + this.arguments = new HashMap<>(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 173d299da6..0792b8240d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -43,6 +43,8 @@ public interface CalculatedFieldState { List getRequiredArguments(); + void setRequiredArguments(List requiredArguments); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 6487ce1a43..18e604118d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -25,19 +25,19 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.RollingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; @@ -92,8 +92,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() .setId(toProto(stateId)) - .setType(state.getType().name()) - .addAllRequiredArguments(state.getRequiredArguments()); + .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { @@ -107,42 +106,21 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { - SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() - .setTs(entry.getTs()); - - if (entry.getVersion() != null) { - singleValueProtoBuilder.setVersion(entry.getVersion()); - } - - KvEntry value = entry.getKvEntryValue(); - if (value != null) { - singleValueProtoBuilder.setHasV(true) - .setValue(ProtoUtils.toKeyValueProto(value)); - } - - return SingleValueArgumentProto.newBuilder() + SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName) - .setValue(singleValueProtoBuilder.build()) - .build(); - } + .setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); - private RollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { - RollingArgumentProto.Builder rollingArgumentProtoBuilder = RollingArgumentProto.newBuilder() - .setArgName(argName); + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); - entry.getTsRecords().forEach((ts, value) -> { - SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() - .setTs(ts); + return builder.build(); + } - if (value != null) { - singleValueProtoBuilder.setHasV(true) - .setValue(ProtoUtils.toKeyValueProto(value)); - } + private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - rollingArgumentProtoBuilder.addValues(singleValueProtoBuilder.build()); - }); + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); - return rollingArgumentProtoBuilder.build(); + return builder.build(); } private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { @@ -153,8 +131,8 @@ public class RocksDBStateService implements CalculatedFieldStateService { CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(proto.getRequiredArgumentsList()); - case SCRIPT -> new ScriptCalculatedFieldState(proto.getRequiredArgumentsList()); + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -162,27 +140,25 @@ public class RocksDBStateService implements CalculatedFieldStateService { if (CalculatedFieldType.SCRIPT.equals(type)) { proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromRollingArgumentProto(argProto))); + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); } return state; } private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { - SingleValueProto valueProto = proto.getValue(); - BasicKvEntry value = valueProto.getHasV() ? ProtoUtils.fromProto(valueProto.getValue()) : null; - - return new SingleValueArgumentEntry(valueProto.getTs(), value, valueProto.getVersion()); + TsValueProto tsValueProto = proto.getValue(); + long ts = tsValueProto.getTs(); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); } - private TsRollingArgumentEntry fromRollingArgumentProto(RollingArgumentProto proto) { + private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { TreeMap tsRecords = new TreeMap<>(); - - proto.getValuesList().forEach(singleValueProto -> { - BasicKvEntry value = singleValueProto.getHasV() ? ProtoUtils.fromProto(singleValueProto.getValue()) : null; - tsRecords.put(singleValueProto.getTs(), value); + proto.getTsValueList().forEach(tsValueProto -> { + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); + tsRecords.put(tsValueProto.getTs(), kvEntry); }); - return new TsRollingArgumentEntry(tsRecords); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 0fd630fead..054adbab1b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,11 +135,11 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; - private long maxCalculatedFieldsPerTenant; - private long maxCalculatedFieldsPerEntity; + private long maxCalculatedFields; private long maxArgumentsPerCF; private long maxDataPointsPerRollingArg; private long maxStateSizeInKBytes; + private long maxSingleValueArgumentSizeInKBytes; @Override public long getProfileThreshold(ApiUsageRecordKey key) { @@ -181,7 +181,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura case DASHBOARD -> maxDashboards; case RULE_CHAIN -> maxRuleChains; case EDGE -> maxEdges; - case CALCULATED_FIELD -> maxCalculatedFieldsPerTenant; + case CALCULATED_FIELD -> maxCalculatedFields; default -> 0; }; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 6ee4108f9a..4a2f3710ae 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -790,7 +790,7 @@ message CalculatedFieldTelemetryMsgProto { message CalculatedFieldLinkedTelemetryMsgProto { CalculatedFieldTelemetryMsgProto msg = 1; - repeated CalculatedFieldEntityCtxIdProto links = 2; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -808,30 +808,18 @@ message CalculatedFieldIdProto { int64 calculatedFieldIdLSB = 2; } -message SingleValueProto { - int64 ts = 1; - int64 version = 2; - bool has_v = 4; - KeyValueProto value = 5; -} - message SingleValueArgumentProto { string argName = 1; - SingleValueProto value = 2; -} - -message RollingArgumentProto { - string argName = 1; - repeated SingleValueProto values = 2; + TsValueProto value = 2; + int64 version = 3; } message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; // int32 version = 2; string type = 3; - repeated string requiredArguments = 4; - repeated SingleValueArgumentProto singleValueArguments = 5; - repeated RollingArgumentProto rollingValueArguments = 6; + repeated SingleValueArgumentProto singleValueArguments = 4; + repeated TsValueListProto rollingValueArguments = 5; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index db8997ceb8..b52c2fe9b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -19,7 +19,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -39,7 +38,6 @@ public class CalculatedFieldDataValidator extends DataValidator @Override protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { validateNumberOfEntitiesPerTenant(tenantId, EntityType.CALCULATED_FIELD); - validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); validateNumberOfArgumentsPerCF(tenantId, calculatedField); } @@ -53,17 +51,11 @@ public class CalculatedFieldDataValidator extends DataValidator return old; } - private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { - long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); - long countCFByEntityId = calculatedFieldDao.countCFByEntityId(tenantId, entityId); - - if (countCFByEntityId == maxCFsPerEntity) { - throw new DataValidationException("Calculated fields per entity limit reached!"); - } - } - private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (maxArgumentsPerCF <= 0) { + return; + } if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { throw new DataValidationException("Calculated field arguments limit reached!"); } From eb3f6b137f9f56f6b60098feae76faa29ba7a534 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:03:20 +0200 Subject: [PATCH 2/8] Calculated fields adjustments, improvements and fixes --- .../calculated-fields-table-config.ts | 4 +- .../calculated-fields-table.component.ts | 4 +- ...lated-field-arguments-table.component.html | 11 +++-- ...lated-field-arguments-table.component.scss | 5 ++ ...culated-field-arguments-table.component.ts | 16 ++++++- .../calculated-field-dialog.component.html | 3 +- .../calculated-field-dialog.component.ts | 7 +-- ...ulated-field-argument-panel.component.html | 2 +- ...lculated-field-argument-panel.component.ts | 47 +++++++++++++++---- .../asset-profile-tabs.component.html | 4 ++ .../pages/asset/asset-tabs.component.html | 4 ++ .../device-profile-tabs.component.html | 4 ++ .../shared/models/calculated-field.models.ts | 1 + 13 files changed, 91 insertions(+), 21 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index d8c02558f8..dc7aeffb1d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -54,7 +54,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(); entityId = input(); + entityName = input(); calculatedFieldsTableConfig: CalculatedFieldsTableConfig; @@ -71,7 +72,8 @@ export class CalculatedFieldsTableComponent { this.durationLeft, this.popoverService, this.destroyRef, - this.renderer + this.renderer, + this.entityName() ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index d1d9998e5a..24d67d8654 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,21 +80,26 @@ -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 73f03dc497..8f8b1d5d81 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,4 +19,9 @@ font-size: 14px; } } + .edit-hint { + .mat-mdc-form-field-error { + font-size: 16px; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 328a82184b..a004e3db6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -21,7 +21,9 @@ import { forwardRef, input, Input, + OnChanges, Renderer2, + SimpleChanges, ViewContainerRef, } from '@angular/core'; import { @@ -70,10 +72,11 @@ import { TbPopoverComponent } from '@shared/components/popover.component'; } ], }) -export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges { @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; calculatedFieldType = input() @@ -84,6 +87,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; @@ -105,6 +110,13 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }); } + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.markAsDirty(); + } + } + registerOnChange(fn: (argumentsObj: Record) => void): void { this.propagateChange = fn; } @@ -137,6 +149,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces calculatedFieldType: this.calculatedFieldType(), buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, + entityName: this.entityName, }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -202,6 +215,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), } : {}), refEntityKey: this.fb.group({ + ...value.refEntityKey, type: [{ value: value.refEntityKey.type, disabled: true }], key: [{ value: value.refEntityKey.key, disabled: true }], }), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ca27ac6fd1..37e96cfd91 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -69,6 +69,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" + [entityName]="data.entityName" [calculatedFieldType]="fieldFormGroup.get('type').value" /> @@ -114,7 +115,7 @@ @if (outputFormGroup.get('type').value === OutputType.Attribute) { - {{ 'calculated-fields.output-type' | translate }} + {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c8b2073309..155a384273 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,6 +36,7 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -59,10 +60,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(configuration.arguments)) + startWith(null), + map(() => Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 56568a7bfe..d274c30b8f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -96,7 +96,7 @@ } - @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 60d79d7bd6..b47d2f073c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -49,6 +49,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() argument: CalculatedFieldArgumentValue; @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -83,6 +84,8 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { readonly ArgumentEntityType = ArgumentEntityType; readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + private currentEntityFilter: EntityFilter; + constructor( private fb: FormBuilder, private cd: ChangeDetectorRef, @@ -107,6 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = this.getCurrentEntityFilter(); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -138,30 +142,53 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { } private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { - let entityId: EntityId; + let entityFilter: EntityFilter; switch (entityType) { case ArgumentEntityType.Current: - entityId = this.entityId + entityFilter = this.currentEntityFilter; break; case ArgumentEntityType.Tenant: - entityId = { - id: this.tenantId, - entityType: EntityType.TENANT + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } - this.entityFilter = { - type: AliasFilterType.singleEntity, - singleEntity: entityId, - }; + this.entityFilter = entityFilter; this.cd.markForCheck(); } + private getCurrentEntityFilter(): EntityFilter { + switch (this.entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: this.entityId, + }; + } + } + private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index 9084301783..f70d4f443b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 4329cd7e3c..2b677dc278 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index aa928687ad..37cfafa9aa 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -69,6 +69,10 @@
+ + + diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 533384e1c4..f73926f62b 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -120,6 +120,7 @@ export interface CalculatedFieldDialogData { entityId: EntityId; debugLimitsConfiguration: string; tenantId: string; + entityName?: string; } export interface ArgumentEntityTypeParams { From d14d0d4e8a8f011df869fedd96c9a09964c60af4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Feb 2025 17:03:25 +0200 Subject: [PATCH 3/8] used limits for state persistence --- .../server/actors/ActorSystemContext.java | 6 ++++++ .../CalculatedFieldEntityMessageProcessor.java | 4 ++-- .../CalculatedFieldManagerMessageProcessor.java | 6 +++--- .../service/cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/DefaultCalculatedFieldCache.java | 4 +++- .../cf/DefaultCalculatedFieldExecutionService.java | 4 ++-- .../service/cf/ctx/CalculatedFieldStateService.java | 3 ++- .../cf/ctx/state/BaseCalculatedFieldState.java | 5 +++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 10 +++++++++- .../service/cf/ctx/state/RocksDBStateService.java | 7 +++++-- .../ctx/state/ScriptCalculatedFieldStateTest.java | 12 ++++++++++-- .../ctx/state/SimpleCalculatedFieldStateTest.java | 13 ++++++++++++- .../common/data/cf/configuration/Argument.java | 6 ++++-- .../server/common/data/cf/configuration/Output.java | 2 ++ .../data/cf/configuration/ReferencedEntityKey.java | 4 ++-- 15 files changed, 66 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..49e2c9c46e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -98,6 +98,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9392e3c02a..f6bc5b2d63 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -200,7 +200,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { @@ -209,7 +209,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index ec86e640d4..31c8eee23c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -95,7 +95,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -220,7 +220,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -248,7 +248,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getId()); List newCfList = new ArrayList<>(oldCfList.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 19f60165cf..393fbd3ec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,7 +43,7 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..c8f9a7e882 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -57,6 +58,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final AssetService assetService; private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ApiLimitService apiLimitService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -116,7 +118,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { if (ctx == null) { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { - ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..03c8f3de0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -265,8 +265,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(stateId, state, callback); + public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(ctx, stateId, state, callback); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index c670e97580..e822d52767 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Map; @@ -24,7 +25,7 @@ public interface CalculatedFieldStateService { Map restoreStates(); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 21105a44cb..86d83a1f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import java.util.ArrayList; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; @@ -34,8 +36,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } public BaseCalculatedFieldState() { - this.requiredArguments = new ArrayList<>(); - this.arguments = new HashMap<>(); + this(new ArrayList<>(), new HashMap<>()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 4600628838..c483c6ab5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -67,7 +69,10 @@ public class CalculatedFieldCtx { private boolean initialized; - public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { this.calculatedField = calculatedField; this.cfId = calculatedField.getId(); @@ -96,6 +101,9 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSizeInKBytes = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes); } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 18e604118d..371751cc6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -59,8 +59,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateProto stateProto = toProto(stateId, state); + if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); + } callback.onSuccess(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 42ff828dbd..bded0d058d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -45,6 +47,8 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; @SpringBootTest(classes = DefaultTbelInvokeService.class) public class ScriptCalculatedFieldStateTest { @@ -64,9 +68,13 @@ public class ScriptCalculatedFieldStateTest { @Autowired private TbelInvokeService tbelInvokeService; + @MockBean + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -219,7 +227,7 @@ public class ScriptCalculatedFieldStateTest { ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); argument1.setRefEntityKey(refEntityKey1); argument1.setLimit(5); - argument1.setTimeWindow(30000); + argument1.setTimeWindow(30000L); Argument argument2 = new Argument(); ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d6b384d85b..d803ad2dab 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -32,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -41,7 +45,10 @@ import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) public class SimpleCalculatedFieldStateTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); @@ -55,9 +62,13 @@ public class SimpleCalculatedFieldStateTest { private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; + @Mock + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), null); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index b61c3bc507..9923751db6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.springframework.lang.Nullable; import org.thingsboard.server.common.data.id.EntityId; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Argument { @Nullable @@ -27,7 +29,7 @@ public class Argument { private ReferencedEntityKey refEntityKey; private String defaultValue; - private int limit; - private long timeWindow; + private Integer limit; + private Long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 12cf97338a..b57b19d3ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b4bcc77a17..fd0bf3ceb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -15,18 +15,18 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) public class ReferencedEntityKey { private String key; private ArgumentType type; private AttributeScope scope; - - } From 555acff6b4ced9405b2ef2cfcc37e90a6b4d8fa4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:12:03 +0200 Subject: [PATCH 4/8] fix --- .../panel/calculated-field-argument-panel.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b47d2f073c..fe60162005 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -173,7 +173,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { switch (this.entityId.entityType) { case EntityType.ASSET_PROFILE: return { - deviceTypes: [this.entityName], + assetTypes: [this.entityName], type: AliasFilterType.assetType }; case EntityType.DEVICE_PROFILE: From d185b784274f9ed19302c6bce1e5976fb29bd3ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:57:05 +0200 Subject: [PATCH 5/8] Resolved PR comments --- ...lated-field-arguments-table.component.html | 19 ++++++++------ ...lated-field-arguments-table.component.scss | 9 ++++--- .../calculated-field-dialog.component.ts | 7 +++--- ...lculated-field-argument-panel.component.ts | 25 +++---------------- .../shared/models/calculated-field.models.ts | 21 ++++++++++++++++ 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 24d67d8654..b25809ac82 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -87,19 +87,24 @@ (click)="manageArgument($event, button, $index)" [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> - edit - @if (argumentsFormArray.dirty - && group.get('refEntityKey').get('type').value === ArgumentType.Rolling - && calculatedFieldType() === CalculatedFieldType.SIMPLE) { - - } + + edit + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8f8b1d5d81..3a1f1fbe25 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,9 +19,10 @@ font-size: 14px; } } - .edit-hint { - .mat-mdc-form-field-error { - font-size: 16px; - } +} + +:host { + .field-action { + color: rgba(0, 0, 0, 0.54); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 155a384273..55fc299475 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,7 +36,6 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; -import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -60,10 +59,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => Object.keys(argumentsObj)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index fe60162005..510bdd95f3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -25,7 +25,8 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - CalculatedFieldType + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, filter } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -110,7 +111,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); - this.currentEntityFilter = this.getCurrentEntityFilter(); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -169,26 +170,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.cd.markForCheck(); } - private getCurrentEntityFilter(): EntityFilter { - switch (this.entityId.entityType) { - case EntityType.ASSET_PROFILE: - return { - assetTypes: [this.entityName], - type: AliasFilterType.assetType - }; - case EntityType.DEVICE_PROFILE: - return { - deviceTypes: [this.entityName], - type: AliasFilterType.deviceType - }; - default: - return { - type: AliasFilterType.singleEntity, - singleEntity: this.entityId, - }; - } - } - private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f73926f62b..fac1e9f942 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -20,6 +20,7 @@ import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { debugSettings?: EntityDebugSettings; @@ -133,3 +134,23 @@ export const ArgumentEntityTypeParamsMap =new Map { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} From 33825e5660e5aeb39271904691a503d5c2232fd2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:03 +0200 Subject: [PATCH 6/8] Changed styling --- ...calculated-field-arguments-table.component.html | 7 +++---- ...calculated-field-arguments-table.component.scss | 14 ++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index b25809ac82..4b7e516db0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 3a1f1fbe25..188371b0d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -13,6 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +:host { + .tb-form-table-row-cell-buttons{ + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + :host ::ng-deep { .tb-inline-field { a { @@ -20,9 +28,3 @@ } } } - -:host { - .field-action { - color: rgba(0, 0, 0, 0.54); - } -} From 1ac9d4137773e539880b6ba51063d8c5ddeac499 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:34 +0200 Subject: [PATCH 7/8] formatting --- .../calculated-field-arguments-table.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 188371b0d8..a0c90bba32 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-form-table-row-cell-buttons{ + .tb-form-table-row-cell-buttons { --mat-badge-legacy-small-size-container-size: 8px; --mat-badge-small-size-container-overlap-offset: -5px; --mat-badge-small-size-text-size: 0; From 3818d1cb68d35129e535d96f2ddb098c59812bdb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Feb 2025 12:34:10 +0200 Subject: [PATCH 8/8] debug events(wip) --- .../server/actors/ActorSystemContext.java | 95 ++++++++++++------- .../cf/ctx/state/RocksDBStateService.java | 3 +- .../permission/TenantAdminPermissions.java | 1 + .../src/main/resources/thingsboard.yml | 6 ++ .../dao/cf/BaseCalculatedFieldService.java | 2 +- .../CalculatedFieldDebugEventRepository.java | 2 +- .../main/resources/sql/schema-entities.sql | 4 +- 7 files changed, 72 insertions(+), 41 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 49e2c9c46e..652cdabf77 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -137,6 +137,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j @Component @@ -179,6 +180,8 @@ public class ActorSystemContext { private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap cfDebugPerTenantLimits = new ConcurrentHashMap<>(); + public ConcurrentMap getDebugPerTenantLimits() { return debugPerTenantLimits; } @@ -441,6 +444,11 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + /** * The following Service will be null if we operate in tb-core mode */ @@ -517,11 +525,6 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; - @Lazy - @Autowired(required = false) - @Getter - private ApiLimitService apiLimitService; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -625,6 +628,14 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean cfDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String cfDebugPerTenantLimitsConfiguration; + @Getter @Setter private TbActorSystem actorSystem; @@ -753,37 +764,6 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString(arguments)); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); - } - } - private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> @@ -817,6 +797,49 @@ public class ActorSystemContext { Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + if (cfDebugPerTenantEnabled) { + TbRateLimits rateLimits = cfDebugPerTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(cfDebugPerTenantLimitsConfiguration)); + + if (rateLimits.tryConsume()) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + if (log.isTraceEnabled()) { + log.trace("[{}] Tenant level debug mode rate limit detected: {}", tenantId, calculatedFieldId); + } + } + } + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 371751cc6f..8a6a5c9cb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -61,7 +61,8 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); - if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { rocksDBService.put(toProto(stateId), toProto(stateId, state)); } callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 5b98a56f24..b0e35e57e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,6 +55,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..197f05a5d5 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index d37a4a53c1..b062a2de5e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -62,9 +62,9 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); - updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java index c0bd21ca74..a759babcfb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -35,7 +35,7 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override - @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + @Query("SELECT e FROM CalculatedFieldDebugEventEntity e WHERE " + "e.tenantId = :tenantId " + "AND e.entityId = :entityId " + "AND (:startTime IS NULL OR e.ts >= :startTime) " + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a6d1e8800d..4b395a5768 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -955,10 +955,10 @@ CREATE TABLE IF NOT EXISTS cf_debug_event ( id uuid NOT NULL, tenant_id uuid NOT NULL , ts bigint NOT NULL, - entity_id uuid NOT NULL, + entity_id uuid NOT NULL, -- calculated field id service_id varchar, cf_id uuid NOT NULL, - e_entity_id uuid, + e_entity_id uuid, -- target entity id e_entity_type varchar, e_msg_id uuid, e_msg_type varchar,