From fdc575c176c676418fdb9ce9a303bbe0a2886c99 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 22 Sep 2025 15:04:44 +0300 Subject: [PATCH 001/122] Structures for new Alarm rules CF --- .../common/data/alarm/rule/AlarmRule.java | 33 ++++++++++++ .../alarm/rule/condition/AlarmCondition.java | 49 +++++++++++++++++ .../rule/condition/AlarmConditionType.java | 22 ++++++++ .../rule/condition/AlarmConditionValue.java | 26 +++++++++ .../condition/DurationAlarmCondition.java | 35 ++++++++++++ .../condition/RepeatingAlarmCondition.java | 32 +++++++++++ .../rule/condition/SimpleAlarmCondition.java | 25 +++++++++ .../expression/AlarmConditionExpression.java | 35 ++++++++++++ .../AlarmConditionExpressionType.java | 21 ++++++++ .../SimpleAlarmConditionExpression.java | 32 +++++++++++ .../TbelAlarmConditionExpression.java | 32 +++++++++++ .../condition/schedule/AlarmSchedule.java | 38 +++++++++++++ .../condition/schedule/AlarmScheduleType.java | 22 ++++++++ .../condition/schedule/AnyTimeSchedule.java | 25 +++++++++ .../schedule/CustomTimeSchedule.java | 33 ++++++++++++ .../schedule/CustomTimeScheduleItem.java | 30 +++++++++++ .../schedule/SpecificTimeSchedule.java | 35 ++++++++++++ .../common/data/cf/CalculatedFieldType.java | 7 +-- .../AlarmCalculatedFieldConfiguration.java | 54 +++++++++++++++++++ ...entsBasedCalculatedFieldConfiguration.java | 12 +++++ .../BaseCalculatedFieldConfiguration.java | 15 ------ .../CFArgumentDynamicSourceType.java | 1 + .../CalculatedFieldConfiguration.java | 8 +-- .../CfArgumentDynamicSourceConfiguration.java | 3 +- ...entCustomerDynamicSourceConfiguration.java | 30 +++++++++++ 25 files changed, 633 insertions(+), 22 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/SimpleAlarmCondition.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpression.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpressionType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/TbelAlarmConditionExpression.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmSchedule.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmScheduleType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AnyTimeSchedule.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeSchedule.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeScheduleItem.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/SpecificTimeSchedule.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java new file mode 100644 index 0000000000..bd4c4b0dd8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmCondition; +import org.thingsboard.server.common.data.id.DashboardId; + +@Data +public class AlarmRule { + + @Valid + @NotNull + private AlarmCondition condition; + private String alarmDetails; + private DashboardId dashboardId; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java new file mode 100644 index 0000000000..36b03b62ae --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.Valid; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.jetbrains.annotations.NotNull; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SIMPLE", value = SimpleAlarmCondition.class), + @Type(name = "DURATION", value = DurationAlarmCondition.class), + @Type(name = "REPEATING", value = RepeatingAlarmCondition.class), +}) +@Data +@NoArgsConstructor +public abstract class AlarmCondition { + + @NotNull + @Valid + private AlarmConditionExpression expression; + private AlarmConditionValue schedule; + + @JsonIgnore + public abstract AlarmConditionType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionType.java new file mode 100644 index 0000000000..fd98ed2984 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +public enum AlarmConditionType { + SIMPLE, + DURATION, + REPEATING +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java new file mode 100644 index 0000000000..4bde76820a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +import lombok.Data; + +@Data +public class AlarmConditionValue { + + private T staticValue; + private String dynamicValueArgument; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java new file mode 100644 index 0000000000..7656d63bc0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.concurrent.TimeUnit; + +@Data +@EqualsAndHashCode(callSuper = true) +public class DurationAlarmCondition extends AlarmCondition { + + private TimeUnit unit; + private AlarmConditionValue value; + + @Override + public AlarmConditionType getType() { + return AlarmConditionType.DURATION; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java new file mode 100644 index 0000000000..9a57bb4631 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class RepeatingAlarmCondition extends AlarmCondition { + + private AlarmConditionValue count; + + @Override + public AlarmConditionType getType() { + return AlarmConditionType.REPEATING; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/SimpleAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/SimpleAlarmCondition.java new file mode 100644 index 0000000000..8e2a7593b0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/SimpleAlarmCondition.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition; + +public class SimpleAlarmCondition extends AlarmCondition { + + @Override + public AlarmConditionType getType() { + return AlarmConditionType.SIMPLE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpression.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpression.java new file mode 100644 index 0000000000..e855f8efd3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpression.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SIMPLE", value = SimpleAlarmConditionExpression.class), + @Type(name = "TBEL", value = TbelAlarmConditionExpression.class), +}) +public interface AlarmConditionExpression { + + @JsonIgnore + AlarmConditionExpressionType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpressionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpressionType.java new file mode 100644 index 0000000000..f0b8f5253d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionExpressionType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +public enum AlarmConditionExpressionType { + SIMPLE, + TBEL +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java new file mode 100644 index 0000000000..fe108c39e1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +public class SimpleAlarmConditionExpression implements AlarmConditionExpression { + + @NotBlank + private String expression; + + @Override + public AlarmConditionExpressionType getType() { + return AlarmConditionExpressionType.SIMPLE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/TbelAlarmConditionExpression.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/TbelAlarmConditionExpression.java new file mode 100644 index 0000000000..50f73e887b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/TbelAlarmConditionExpression.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +public class TbelAlarmConditionExpression implements AlarmConditionExpression { + + @NotBlank + private String expression; + + @Override + public AlarmConditionExpressionType getType() { + return AlarmConditionExpressionType.TBEL; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmSchedule.java new file mode 100644 index 0000000000..e7394c94bd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmSchedule.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(value = AnyTimeSchedule.class, name = "ANY_TIME"), + @Type(value = SpecificTimeSchedule.class, name = "SPECIFIC_TIME"), + @Type(value = CustomTimeSchedule.class, name = "CUSTOM") +}) +public interface AlarmSchedule extends Serializable { + + @JsonIgnore + AlarmScheduleType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmScheduleType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmScheduleType.java new file mode 100644 index 0000000000..d18d92834e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AlarmScheduleType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +public enum AlarmScheduleType { + ANY_TIME, + SPECIFIC_TIME, + CUSTOM +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AnyTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AnyTimeSchedule.java new file mode 100644 index 0000000000..e84f767f5b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/AnyTimeSchedule.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +public class AnyTimeSchedule implements AlarmSchedule { + + @Override + public AlarmScheduleType getType() { + return AlarmScheduleType.ANY_TIME; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeSchedule.java new file mode 100644 index 0000000000..b084494d28 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeSchedule.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +import lombok.Data; + +import java.util.List; + +@Data +public class CustomTimeSchedule implements AlarmSchedule { + + private String timezone; + private List items; + + @Override + public AlarmScheduleType getType() { + return AlarmScheduleType.CUSTOM; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeScheduleItem.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeScheduleItem.java new file mode 100644 index 0000000000..8a2bb97c39 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/CustomTimeScheduleItem.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class CustomTimeScheduleItem implements Serializable { + + private boolean enabled; + private int dayOfWeek; + private long startsOn; + private long endsOn; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/SpecificTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/SpecificTimeSchedule.java new file mode 100644 index 0000000000..7242d2c9cd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/schedule/SpecificTimeSchedule.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.schedule; + +import lombok.Data; + +import java.util.Set; + +@Data +public class SpecificTimeSchedule implements AlarmSchedule { + + private String timezone; + private Set daysOfWeek; + private long startsOn; + private long endsOn; + + @Override + public AlarmScheduleType getType() { + return AlarmScheduleType.SPECIFIC_TIME; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index d4dd2c5812..3399808a35 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -16,7 +16,8 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - - SIMPLE, SCRIPT, GEOFENCING - + SIMPLE, + SCRIPT, + GEOFENCING, + ALARM } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..bb0834b3a7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.List; +import java.util.Map; + +@Data +public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + + private Map arguments; + + private Map createRules; + private AlarmRule clearRule; + + private boolean propagate; + private boolean propagateToOwner; + private boolean propagateToTenant; + private List propagateRelationTypes; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.ALARM; + } + + @Override + public Output getOutput() { + return null; + } + + @Override + public void validate() { + + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java index 225278e776..31c95b2119 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java @@ -15,10 +15,22 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; public interface ArgumentsBasedCalculatedFieldConfiguration extends CalculatedFieldConfiguration { Map getArguments(); + default List getReferencedEntities() { + return getArguments().values().stream() + .map(Argument::getRefEntityId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index c270874605..535febf3a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -16,15 +16,8 @@ package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; @Data public abstract class BaseCalculatedFieldConfiguration implements ExpressionBasedCalculatedFieldConfiguration { @@ -33,14 +26,6 @@ public abstract class BaseCalculatedFieldConfiguration implements ExpressionBase protected String expression; protected Output output; - @Override - public List getReferencedEntities() { - return arguments.values().stream() - .map(Argument::getRefEntityId) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - @Override public void validate() { if (arguments.containsKey("ctx")) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java index bd2e9b0c00..e8ef6c7835 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.cf.configuration; public enum CFArgumentDynamicSourceType { + CURRENT_CUSTOMER, RELATION_QUERY } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 972a3e0ee9..7b608192db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -36,9 +37,10 @@ import java.util.stream.Collectors; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING") + @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), + @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java index f36071615e..c16d8abfcc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java @@ -26,7 +26,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = RelationQueryDynamicSourceConfiguration.class, name = "RELATION_QUERY") + @JsonSubTypes.Type(value = RelationQueryDynamicSourceConfiguration.class, name = "RELATION_QUERY"), + @JsonSubTypes.Type(value = CurrentCustomerDynamicSourceConfiguration.class, name = "CURRENT_CUSTOMER") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CfArgumentDynamicSourceConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java new file mode 100644 index 0000000000..8ede2c28df --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; + +@Data +public class CurrentCustomerDynamicSourceConfiguration implements CfArgumentDynamicSourceConfiguration { + + private boolean inherit; // TODO: implement + + @Override + public CFArgumentDynamicSourceType getType() { + return CFArgumentDynamicSourceType.CURRENT_CUSTOMER; + } + +} From 3e11282d8f0bb9b507f6196f47348a384177a0eb Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 22 Sep 2025 16:24:18 +0300 Subject: [PATCH 002/122] Base implementation of Alarm rules CF --- .../server/actors/app/AppActor.java | 30 +- .../CalculatedFieldAlarmActionMsg.java | 41 +++ .../CalculatedFieldEntityActionEventMsg.java | 57 ++++ .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 81 +++-- .../CalculatedFieldLinkedTelemetryMsg.java | 3 +- .../CalculatedFieldManagerActor.java | 3 + ...alculatedFieldManagerMessageProcessor.java | 62 +++- .../CalculatedFieldTelemetryMsg.java | 2 +- .../EntityInitCalculatedFieldMsg.java | 13 +- .../server/actors/tenant/TenantActor.java | 10 +- ...tractCalculatedFieldProcessingService.java | 18 +- .../AbstractCalculatedFieldStateService.java | 2 +- .../cf/AlarmCalculatedFieldResult.java | 80 +++++ .../service/cf/CalculatedFieldCache.java | 3 + .../cf/CalculatedFieldProcessingService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 27 +- .../cf/DefaultCalculatedFieldCache.java | 48 ++- ...faultCalculatedFieldProcessingService.java | 22 +- .../DefaultCalculatedFieldQueueService.java | 31 +- .../cf/TelemetryCalculatedFieldResult.java | 74 ++++ .../ctx/state/BaseCalculatedFieldState.java | 39 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 179 +++++++--- .../cf/ctx/state/CalculatedFieldState.java | 25 +- .../ctx/state/ScriptCalculatedFieldState.java | 46 +-- .../ctx/state/SimpleCalculatedFieldState.java | 40 +-- .../alarm/AlarmCalculatedFieldState.java | 320 ++++++++++++++++++ .../cf/ctx/state/alarm/AlarmEvalResult.java | 6 +- .../cf/ctx/state/alarm/AlarmRuleState.java | 233 +++++++++++++ .../GeofencingCalculatedFieldState.java | 39 ++- .../edge/RelatedEdgesSourcingListener.java | 20 +- .../processor/alarm/BaseAlarmProcessor.java | 2 +- .../entitiy/EntityStateSourcingListener.java | 75 +++- ...faultTbCalculatedFieldConsumerService.java | 35 +- .../DefaultAlarmSubscriptionService.java | 7 +- .../DefaultTelemetrySubscriptionService.java | 1 + .../utils/CalculatedFieldArgumentUtils.java | 13 +- .../server/utils/CalculatedFieldUtils.java | 67 +++- .../thingsboard/server/cf/AlarmRulesTest.java | 191 +++++++++++ .../cf/CalculatedFieldIntegrationTest.java | 23 +- .../AbstractRuleEngineControllerTest.java | 18 - .../server/controller/AbstractWebTest.java | 24 ++ ...actRuleEngineLifecycleIntegrationTest.java | 2 +- .../GeofencingCalculatedFieldStateTest.java | 48 +-- .../state/ScriptCalculatedFieldStateTest.java | 26 +- .../state/SimpleCalculatedFieldStateTest.java | 28 +- .../utils/CalculatedFieldUtilsTest.java | 7 +- .../server/dao/alarm/AlarmService.java | 2 +- .../server/common/msg/MsgType.java | 2 + .../msg/ToCalculatedFieldSystemMsg.java | 5 - .../common/msg/aware/TenantAwareMsg.java | 7 +- common/proto/src/main/proto/queue.proto | 23 ++ ...unctionsUtil.java => ExpressionUtils.java} | 14 +- .../org/thingsboard/common/util/KvUtil.java | 31 ++ .../server/dao/alarm/BaseAlarmService.java | 24 +- .../server/dao/service/AlarmServiceTest.java | 12 +- .../engine/api/RuleEngineAlarmService.java | 2 + .../rule/engine/action/TbAlarmResult.java | 2 + .../rule/engine/math/TbMathNode.java | 10 +- .../rule/engine/profile/AlarmRuleState.java | 17 +- .../profile/TbDeviceProfileNodeTest.java | 4 - 61 files changed, 1811 insertions(+), 473 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldAlarmActionMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActionEventMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java rename rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmStateUpdateResult.java => application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java (82%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java create mode 100644 application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java rename common/util/src/main/java/org/thingsboard/common/util/{ExpressionFunctionsUtil.java => ExpressionUtils.java} (87%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 27bdec2422..e515d58695 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -43,7 +43,6 @@ import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import java.util.HashSet; import java.util.Optional; @@ -94,11 +93,14 @@ public class AppActor extends ContextAwareActor { case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; + case CF_ENTITY_ACTION_EVENT_MSG: + forwardToTenantActor((TenantAwareMsg) msg, true); + break; case QUEUE_TO_RULE_ENGINE_MSG: onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: - onToDeviceActorMsg((TenantAwareMsg) msg, false); + forwardToTenantActor((TenantAwareMsg) msg, false); break; case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_CREDENTIALS_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -108,7 +110,7 @@ public class AppActor extends ContextAwareActor { case DEVICE_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: case REMOVE_RPC_TO_DEVICE_ACTOR_MSG: - onToDeviceActorMsg((TenantAwareMsg) msg, true); + forwardToTenantActor((TenantAwareMsg) msg, true); break; case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); @@ -117,11 +119,11 @@ public class AppActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. // same for the Linked telemetry. - onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + forwardToTenantActor((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + forwardToTenantActor((ToCalculatedFieldSystemMsg) msg, false); break; default: return false; @@ -187,7 +189,7 @@ public class AppActor extends ContextAwareActor { } } - private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + private void forwardToTenantActor(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { tenantActor.tellWithHighPriority(msg); @@ -199,21 +201,6 @@ public class AppActor extends ContextAwareActor { }); } - - private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { - getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { - if (priority) { - tenantActor.tellWithHighPriority(msg); - } else { - tenantActor.tell(msg); - } - }, () -> { - if (msg instanceof TransportToDeviceActorMsgWrapper) { - ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); - } - }); - } - private Optional getOrCreateTenantActor(TenantId tenantId) { if (deletedTenants.contains(tenantId)) { return Optional.empty(); @@ -245,6 +232,7 @@ public class AppActor extends ContextAwareActor { public TbActor createActor() { return new AppActor(context); } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldAlarmActionMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldAlarmActionMsg.java new file mode 100644 index 0000000000..3202296345 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldAlarmActionMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +@Builder +public class CalculatedFieldAlarmActionMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final Alarm alarm; + private final ActionType action; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ALARM_ACTION_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActionEventMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActionEventMsg.java new file mode 100644 index 0000000000..6fc191e3db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActionEventMsg.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; + +@Data +@Builder +public class CalculatedFieldEntityActionEventMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final JsonNode entity; + private final ActionType action; + private final TbCallback callback; + + public static CalculatedFieldEntityActionEventMsg fromProto(EntityActionEventProto proto, + TbCallback callback) { + return CalculatedFieldEntityActionEventMsg.builder() + .tenantId((TenantId) ProtoUtils.fromProto(proto.getTenantId())) + .entityId(ProtoUtils.fromProto(proto.getEntityId())) + .entity(JacksonUtil.toJsonNode(proto.getEntity())) + .action(ActionType.valueOf(proto.getAction())) + .callback(callback) + .build(); + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_ACTION_EVENT_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index 2a5f3c3cfd..bf24c8ff84 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -78,6 +78,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DYNAMIC_ARGUMENTS_REFRESH_MSG: processor.process((EntityCalculatedFieldDynamicArgumentsRefreshMsg) msg); break; + case CF_ALARM_ACTION_MSG: + processor.process((CalculatedFieldAlarmActionMsg) msg); + break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 7513ca41e2..77665a1f10 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -21,10 +21,12 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -48,6 +50,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.ArrayList; @@ -63,6 +66,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; /** * @author Andrew Shvayka @@ -120,12 +124,23 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); var ctx = msg.getCtx(); - if (msg.isForceReinit()) { - log.debug("Force reinitialization of CF: [{}].", ctx.getCfId()); + CalculatedFieldState state; + if (msg.getStateAction() == StateAction.RECREATE) { states.remove(ctx.getCfId()); + state = null; + } else { + state = states.get(ctx.getCfId()); } try { - var state = getOrInitState(ctx); + if (state == null) { + state = createState(ctx); + } else if (msg.getStateAction() == StateAction.REINIT) { + log.debug("Force reinitialization of CF: [{}].", ctx.getCfId()); + state.reset(ctx); + initState(state, ctx); + } else { + state.init(ctx); + } if (state.isSizeOk()) { processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); } else { @@ -239,6 +254,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); } + public void process(CalculatedFieldAlarmActionMsg msg) { + log.debug("[{}] Processing alarm action event msg: {}", entityId, msg); + states.values().forEach(state -> { + if (state instanceof AlarmCalculatedFieldState alarmCfState) { + Alarm stateAlarm = alarmCfState.getCurrentAlarm(); + if (stateAlarm != null && stateAlarm.getId().equals(msg.getAlarm().getId())) { + alarmCfState.processAlarmAction(msg.getAlarm(), msg.getAction()); + } + } + }); + msg.getCallback().onSuccess(); + } + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @@ -264,7 +292,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM CalculatedFieldState state = states.get(ctx.getCfId()); boolean justRestored = false; if (state == null) { - state = getOrInitState(ctx); + state = createState(ctx); justRestored = true; } else if (state.isDirty()) { log.debug("[{}][{}] Going to update dirty CF state.", entityId, ctx.getCfId()); @@ -277,7 +305,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } if (state.isSizeOk()) { - if (state.updateState(ctx, newArgValues) || justRestored) { + if (state.update(ctx, newArgValues) || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); @@ -289,30 +317,38 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - @SneakyThrows - private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { - CalculatedFieldState state = states.get(ctx.getCfId()); - if (state != null) { - return state; - } else { - ListenableFuture stateFuture = cfService.fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); - states.put(ctx.getCfId(), state); - } + private CalculatedFieldState createState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = createStateByType(ctx, entityId); + initState(state, ctx); return state; } + private void initState(CalculatedFieldState state, CalculatedFieldCtx ctx) { + state.init(ctx); + + Map arguments = fetchArguments(ctx); + state.update(ctx, arguments); + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + states.put(ctx.getCfId(), state); + } + + @SneakyThrows + private Map fetchArguments(CalculatedFieldCtx ctx) { + ListenableFuture> argumentsFuture = cfService.fetchArguments(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentsFuture.get(1, TimeUnit.MINUTES); + } + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeChecked = false; try { if (ctx.isInitialized() && state.isReady()) { - CalculatedFieldResult calculationResult = state.performCalculation(entityId, ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeChecked = true; if (state.isSizeOk()) { @@ -322,13 +358,14 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(); } if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, calculationResult.toStringOrElseNull(), null); + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, calculationResult.stringValue(), null); } } } else { callback.onSuccess(); } } catch (Exception e) { + log.debug("[{}][{}] Failed to process CF state", entityId, ctx.getCfId(), e); throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } finally { if (!stateSizeChecked) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java index 3e0fba2627..f92ed2ca9e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @Data public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { @@ -32,9 +31,9 @@ public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSyste private final CalculatedFieldLinkedTelemetryMsgProto proto; private final TbCallback callback; - @Override public MsgType getMsgType() { return MsgType.CF_LINKED_TELEMETRY_MSG; } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index ab6cb34176..37864d4146 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -73,6 +73,9 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { case CF_ENTITY_LIFECYCLE_MSG: processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); break; + case CF_ENTITY_ACTION_EVENT_MSG: + processor.onEntityActionEventMsg((CalculatedFieldEntityActionEventMsg) msg); + break; case CF_TELEMETRY_MSG: processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 75ca0b4c9b..43a21b196a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,15 +16,18 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; @@ -127,11 +130,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { var cfId = msg.getId().cfId(); - var calculatedField = calculatedFields.get(cfId); + var ctx = calculatedFields.get(cfId); - if (calculatedField != null) { + if (ctx != null) { if (msg.getState() != null) { - msg.getState().setRequiredArguments(calculatedField.getArgNames()); + msg.getState().init(ctx); } log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); @@ -198,6 +201,22 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + public void onEntityActionEventMsg(CalculatedFieldEntityActionEventMsg msg) { + switch (msg.getAction()) { + case ALARM_ACK, ALARM_CLEAR, ALARM_DELETE -> { + Alarm alarm = JacksonUtil.treeToValue(msg.getEntity(), Alarm.class); + CalculatedFieldAlarmActionMsg alarmActionMsg = CalculatedFieldAlarmActionMsg.builder() + .tenantId(tenantId) + .alarm(alarm) + .action(msg.getAction()) + .callback(msg.getCallback()) + .build(); + getOrCreateActor(alarm.getOriginator()).tellWithHighPriority(alarmActionMsg); + } + default -> msg.getCallback().onSuccess(); + } + } + private void onProfileDeleted(ComponentLifecycleMsg msg, TbCallback callback) { entityProfileCache.removeProfileId(msg.getEntityId()); callback.onSuccess(); @@ -217,8 +236,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); - entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); - profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { callback.onSuccess(); } @@ -237,7 +256,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); - newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { callback.onSuccess(); } @@ -275,13 +294,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); addLinks(cf); scheduleDynamicArgumentsRefreshTaskForCfIfNeeded(cfCtx); - applyToTargetCfEntityActors(cfCtx, callback, (id, cb) -> initCfForEntity(id, cfCtx, false, cb)); + applyToTargetCfEntityActors(cfCtx, callback, (id, cb) -> initCfForEntity(id, cfCtx, StateAction.INIT, cb)); } } } private CalculatedFieldCtx getCfCtx(CalculatedField cf) { - return new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService(), systemContext.getRelationService()); + return new CalculatedFieldCtx(cf, systemContext); } private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { @@ -295,7 +314,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = getCfCtx(newCf); + var newCfCtx = getCfCtx(newCf); // fixme wtf? why isn't oldCfCtx closed properly? when to close it? try { newCfCtx.init(); } catch (Exception e) { @@ -328,21 +347,28 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware deleteLinks(oldCfCtx); addLinks(newCf); - // We use copy on write lists to safely pass the reference to another actor for the iteration. - // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); - if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { - applyToTargetCfEntityActors(newCfCtx, callback, (id, cb) -> initCfForEntity(id, newCfCtx, stateChanges, cb)); + StateAction stateAction; + if (newCfCtx.getCfType() != oldCfCtx.getCfType()) { + stateAction = StateAction.RECREATE; + } else if (newCfCtx.hasStateChanges(oldCfCtx)) { + stateAction = StateAction.REINIT; + } else if (newCfCtx.hasContextOnlyChanges(oldCfCtx)) { + stateAction = StateAction.REPROCESS; } else { callback.onSuccess(); + return; } + + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + applyToTargetCfEntityActors(newCfCtx, callback, (id, cb) -> initCfForEntity(id, newCfCtx, stateAction, cb)); } } } private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var cfCtx = calculatedFields.remove(cfId); + var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -489,9 +515,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); } - private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, StateAction stateAction, TbCallback callback) { log.debug("Pushing entity init CF msg to specific actor [{}]", entityId); - getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, stateAction, callback)); } private boolean isMyPartition(EntityId entityId, TbCallback callback) { @@ -555,7 +581,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void initCalculatedField(CalculatedField cf) throws CalculatedFieldException { - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService(), systemContext.getRelationService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext); try { cfCtx.init(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java index 68cd149cdf..a174cff268 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -31,9 +31,9 @@ public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { private final CalculatedFieldTelemetryMsgProto proto; private final TbCallback callback; - @Override public MsgType getMsgType() { return MsgType.CF_TELEMETRY_MSG; } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java index 1e8990ff8d..1e0025988d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -16,26 +16,29 @@ package org.thingsboard.server.actors.calculatedField; import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.List; - @Data public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; private final CalculatedFieldCtx ctx; + private final StateAction stateAction; private final TbCallback callback; - private final boolean forceReinit; @Override public MsgType getMsgType() { return MsgType.CF_ENTITY_INIT_CF_MSG; } + + public enum StateAction { + INIT, + REINIT, + RECREATE, + REPROCESS + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 91d25a633b..35cbf3ccd8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; @@ -155,6 +156,9 @@ public class TenantActor extends RuleChainManagerActor { case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; + case CF_ENTITY_ACTION_EVENT_MSG: + forwardToCfActor((TenantAwareMsg) msg, true); + break; case QUEUE_TO_RULE_ENGINE_MSG: onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; @@ -182,11 +186,11 @@ public class TenantActor extends RuleChainManagerActor { case CF_CACHE_INIT_MSG: case CF_STATE_RESTORE_MSG: case CF_PARTITIONS_CHANGE_MSG: - onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + forwardToCfActor((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + forwardToCfActor((ToCalculatedFieldSystemMsg) msg, false); break; default: return false; @@ -194,7 +198,7 @@ public class TenantActor extends RuleChainManagerActor { return true; } - private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + private void forwardToCfActor(TenantAwareMsg msg, boolean priority) { if (cfActor == null) { if (msg instanceof CalculatedFieldStateRestoreMsg) { log.warn("[{}] CF Actor is not initialized. ToCalculatedFieldSystemMsg: [{}]", tenantId, msg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 45305ca9e3..b3632e7f26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -44,7 +44,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.HashMap; import java.util.List; @@ -57,12 +56,11 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; -import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @Slf4j -public abstract class AbstractCalculatedFieldProcessingService { +public abstract class AbstractCalculatedFieldProcessingService implements CalculatedFieldProcessingService { protected final AttributesService attributesService; protected final TimeseriesService timeseriesService; @@ -86,10 +84,11 @@ public abstract class AbstractCalculatedFieldProcessingService { protected abstract String getExecutorNamePrefix(); - public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + @Override + public ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { Map> argFutures = switch (ctx.getCalculatedField().getType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false); - case SIMPLE, SCRIPT -> { + case SIMPLE, SCRIPT, ALARM -> { Map> futures = new HashMap<>(); for (var entry : ctx.getArguments().entrySet()) { var argEntityId = resolveEntityId(entityId, entry.getValue()); @@ -99,11 +98,9 @@ public abstract class AbstractCalculatedFieldProcessingService { yield futures; } }; - return Futures.whenAllComplete(argFutures.values()).call(() -> { - var result = createStateByType(ctx); - result.updateState(ctx, resolveArgumentFutures(argFutures)); - return result; - }, MoreExecutors.directExecutor()); + return Futures.whenAllComplete(argFutures.values()) + .call(() -> resolveArgumentFutures(argFutures), + MoreExecutors.directExecutor()); } protected EntityId resolveEntityId(EntityId entityId, Argument argument) { @@ -174,6 +171,7 @@ public abstract class AbstractCalculatedFieldProcessingService { yield Futures.transform(relationService.findByQuery(tenantId, configuration.toEntityRelationsQuery(entityId)), configuration::resolveEntityIds, calculatedFieldCallbackExecutor); } + case CURRENT_CUSTOMER -> throw new UnsupportedOperationException(); // fixme implement }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index 70b41f069e..a77eb71343 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -64,7 +64,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF protected void processRestoredState(CalculatedFieldStateProto stateMsg) { var id = fromProto(stateMsg.getId()); - var state = fromProto(stateMsg); + var state = fromProto(id, stateMsg); processRestoredState(id, state); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java new file mode 100644 index 0000000000..de48d05630 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.action.TbAlarmResult; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; + +import java.util.List; + +@Data +@Builder +public class AlarmCalculatedFieldResult implements CalculatedFieldResult { + + private final TbAlarmResult alarmResult; + private final AlarmRuleState alarmRuleState; + + @Override + public TbMsg toTbMsg(EntityId entityId, List cfIds) { + TbMsgMetaData metaData = new TbMsgMetaData(); + if (alarmResult.isCreated()) { + metaData.putValue(DataConstants.IS_NEW_ALARM, Boolean.TRUE.toString()); + } else if (alarmResult.isUpdated()) { + metaData.putValue(DataConstants.IS_EXISTING_ALARM, Boolean.TRUE.toString()); + } else if (alarmResult.isSeverityUpdated()) { + metaData.putValue(DataConstants.IS_EXISTING_ALARM, Boolean.TRUE.toString()); + metaData.putValue(DataConstants.IS_SEVERITY_UPDATED_ALARM, Boolean.TRUE.toString()); + } else { + metaData.putValue(DataConstants.IS_CLEARED_ALARM, Boolean.TRUE.toString()); + } + switch (alarmRuleState.getCondition().getType()) { + case REPEATING -> { + metaData.putValue(DataConstants.ALARM_CONDITION_REPEATS, String.valueOf(alarmRuleState.getEventCount())); + } + case DURATION -> { + // TODO: schedule instead of duration + metaData.putValue(DataConstants.ALARM_CONDITION_DURATION, String.valueOf(alarmRuleState.getDuration())); + } + } + + return TbMsg.newMsg() + .type(TbMsgType.ALARM) + .originator(entityId) + .data(JacksonUtil.toString(alarmResult.getAlarm())) + .metaData(metaData) + .build(); + } + + @Override + public String stringValue() { + return alarmResult != null ? JacksonUtil.toString(alarmResult) : null; + } + + @Override + public boolean isEmpty() { + return alarmResult == null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index fb63432fed..5aac75a7c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; +import java.util.function.Predicate; public interface CalculatedFieldCache { @@ -36,6 +37,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); + boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 86ed174485..a9139572b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -25,20 +25,19 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; import java.util.Map; public interface CalculatedFieldProcessingService { - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index c779c27419..c62d5dc6d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -15,27 +15,18 @@ */ package org.thingsboard.server.service.cf; -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.TbMsg; -@Data -public final class CalculatedFieldResult { +import java.util.List; - private final OutputType type; - private final AttributeScope scope; - private final JsonNode result; +public interface CalculatedFieldResult { - public boolean isEmpty() { - return result == null || result.isMissingNode() || result.isNull() || - (result.isObject() && result.isEmpty()) || - (result.isArray() && result.isEmpty()) || - (result.isTextual() && result.asText().isEmpty()); - } + TbMsg toTbMsg(EntityId entityId, List cfIds); - public String toStringOrElseNull() { - return result == null ? null : result.toString(); - } + String stringValue(); + + boolean isEmpty(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index dfe30a0e55..f40aa503f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -19,21 +19,24 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.ConcurrentReferenceHashMap; -import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; @@ -42,6 +45,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Predicate; @Service @Slf4j @@ -51,9 +55,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentReferenceHashMap calculatedFieldFetchLocks = new ConcurrentReferenceHashMap<>(); private final CalculatedFieldService calculatedFieldService; - private final TbelInvokeService tbelInvokeService; - private final ApiLimitService apiLimitService; - private final RelationService relationService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + @Lazy + private final ActorSystemContext systemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -113,7 +118,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { if (ctx == null) { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { - ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService, relationService); + ctx = new CalculatedFieldCtx(calculatedField, systemContext); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } @@ -136,6 +141,27 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { + List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); + for (CalculatedFieldCtx ctx : entityCfs) { + if (filter.test(ctx)) { + return true; + } + } + + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + List profileCfs = getCalculatedFieldCtxsByEntityId(profileId); + for (CalculatedFieldCtx ctx : profileCfs) { + if (filter.test(ctx)) { + return true; + } + } + } + return false; + } + @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { Lock lock = getFetchLock(calculatedFieldId); @@ -185,6 +211,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); } + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + private Lock getFetchLock(CalculatedFieldId id) { return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 17dca5dd64..dfc32741c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -25,13 +25,10 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -51,7 +48,6 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.ArrayList; import java.util.HashMap; @@ -59,13 +55,12 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @TbRuleEngineComponent @Service @Slf4j -public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedFieldProcessingService implements CalculatedFieldProcessingService { +public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedFieldProcessingService { private final TbClusterService clusterService; private final PartitionService partitionService; @@ -86,11 +81,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return "calculated-field-callback"; } - @Override - public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchStateFromDb(ctx, entityId); - } - @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only geofencing calculated fields supports dynamic arguments scheduled updates @@ -115,12 +105,9 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback) { try { - OutputType type = calculatedFieldResult.getType(); - TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(calculatedFieldResult.toStringOrElseNull()).build(); + TbMsg msg = result.toTbMsg(entityId, cfIds); clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { @@ -134,7 +121,7 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } }); } catch (Exception e) { - log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, result, e); callback.onFailure(e); } } @@ -208,6 +195,7 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF public void onFailure(Throwable t) { callback.onFailure(t); } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index fc5d75be56..a3e50812fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -26,9 +26,7 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -45,8 +43,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.EnumSet; @@ -74,8 +70,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } }; - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; @@ -158,21 +152,9 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS if (!supportedReferencedEntities.contains(entityId.getEntityType())) { return false; } - List entityCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId); - for (CalculatedFieldCtx ctx : entityCfs) { - if (filter.test(ctx)) { - return true; - } - } - EntityId profileId = getProfileId(tenantId, entityId); - if (profileId != null) { - List profileCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(profileId); - for (CalculatedFieldCtx ctx : profileCfs) { - if (filter.test(ctx)) { - return true; - } - } + if (calculatedFieldCache.hasCalculatedFields(tenantId, entityId, filter)) { + return true; } List links = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId); @@ -186,14 +168,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return false; } - private EntityId getProfileId(TenantId tenantId, EntityId entityId) { - return switch (entityId.getEntityType()) { - case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> null; - }; - } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); @@ -305,6 +279,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void onFailure(Throwable t) { callback.onFailure(t); } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java new file mode 100644 index 0000000000..1ad666eac5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; + +import java.util.List; +import java.util.Map; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; + +@Data +@Builder +public final class TelemetryCalculatedFieldResult implements CalculatedFieldResult { + + private final OutputType type; + private final AttributeScope scope; + private final JsonNode result; + + @Override + public TbMsg toTbMsg(EntityId entityId, List cfIds) { + TbMsgType msgType = switch (type) { + case ATTRIBUTES -> TbMsgType.POST_ATTRIBUTES_REQUEST; + case TIME_SERIES -> TbMsgType.POST_TELEMETRY_REQUEST; + }; + TbMsgMetaData metaData = switch (type) { + case ATTRIBUTES -> new TbMsgMetaData(Map.of(SCOPE, scope.name())); + case TIME_SERIES -> TbMsgMetaData.EMPTY; + }; + return TbMsg.newMsg() + .type(msgType) + .originator(entityId) + .previousCalculatedFieldIds(cfIds) + .data(stringValue()) + .metaData(metaData) + .build(); + } + + @Override + public String stringValue() { + return result == null ? null : result.toString(); + } + + @Override + public boolean isEmpty() { + return result == null || result.isMissingNode() || result.isNull() || + (result.isObject() && result.isEmpty()) || + (result.isArray() && result.isEmpty()) || + (result.isTextual() && result.asText().isEmpty()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 9a1d06cf24..bd6d5b1a51 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,41 +15,36 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.AllArgsConstructor; -import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -@Data -@AllArgsConstructor +@Getter public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + protected final EntityId entityId; protected List requiredArguments; - protected Map arguments; - protected boolean sizeExceedsLimit; + protected Map arguments = new HashMap<>(); + protected boolean sizeExceedsLimit; protected long latestTimestamp = -1; - public BaseCalculatedFieldState(List requiredArguments) { - this.requiredArguments = requiredArguments; - this.arguments = new HashMap<>(); + public BaseCalculatedFieldState(EntityId entityId) { + this.entityId = entityId; } - public BaseCalculatedFieldState() { - this(new ArrayList<>(), new HashMap<>(), false, -1); + @Override + public void init(CalculatedFieldCtx ctx) { + this.requiredArguments = ctx.getArgNames(); } @Override - public boolean updateState(CalculatedFieldCtx ctx, Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - + public boolean update(CalculatedFieldCtx ctx, Map argumentValues) { boolean stateUpdated = false; for (Map.Entry entry : argumentValues.entrySet()) { @@ -79,10 +74,18 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { return stateUpdated; } + @Override + public void reset(CalculatedFieldCtx ctx) { // must reset everything dependent on arguments + requiredArguments = null; + arguments.clear(); + sizeExceedsLimit = false; + latestTimestamp = -1; + } + @Override public boolean isReady() { return arguments.keySet().containsAll(requiredArguments) && - arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c2cc083853..564e573765 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -15,14 +15,22 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; -import net.objecthunter.exp4j.ExpressionBuilder; import org.mvel2.MVEL; +import org.thingsboard.common.util.ExpressionUtils; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfCtx; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; @@ -36,20 +44,22 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - -import static org.thingsboard.common.util.ExpressionFunctionsUtil.userDefinedFunctions; +import java.util.Objects; +import java.util.stream.Stream; @Data public class CalculatedFieldCtx { @@ -67,10 +77,13 @@ public class CalculatedFieldCtx { private Output output; private String expression; private boolean useLatestTs; + private TbelInvokeService tbelInvokeService; private RelationService relationService; - private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private ThreadLocal customExpression; + private AlarmSubscriptionService alarmService; + + private Map tbelExpressions; + private Map> simpleExpressions; private boolean initialized; @@ -81,7 +94,8 @@ public class CalculatedFieldCtx { private List mainEntityGeofencingArgumentNames; private List linkedEntityGeofencingArgumentNames; - public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService, RelationService relationService) { + public CalculatedFieldCtx(CalculatedField calculatedField, + ActorSystemContext systemContext) { this.calculatedField = calculatedField; this.cfId = calculatedField.getId(); @@ -126,19 +140,20 @@ public class CalculatedFieldCtx { }); } } - this.tbelInvokeService = tbelInvokeService; - this.relationService = relationService; + this.tbelInvokeService = systemContext.getTbelInvokeService(); + this.relationService = systemContext.getRelationService(); + this.alarmService = systemContext.getAlarmService(); - this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; - this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; + this.maxDataPointsPerRollingArg = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); // fixme why tenant profile update is not handled?? + this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; + this.maxSingleValueArgumentSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; } public void init() { switch (cfType) { case SCRIPT -> { try { - this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initTbelExpression(expression); initialized = true; } catch (Exception e) { throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); @@ -146,28 +161,89 @@ public class CalculatedFieldCtx { } case GEOFENCING -> initialized = true; case SIMPLE -> { - if (isValidExpression(expression)) { - this.customExpression = ThreadLocal.withInitial(() -> - new ExpressionBuilder(expression) - .functions(userDefinedFunctions) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build() - ); - initialized = true; - } else { - throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + initSimpleExpression(expression); + initialized = true; + } + case ALARM -> { + AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + Stream rules = configuration.getCreateRules().values().stream(); + if (configuration.getClearRule() != null) { + rules = Stream.concat(rules, Stream.of(configuration.getClearRule())); } + rules.map(rule -> rule.getCondition().getExpression()).forEach(expression -> { + if (expression instanceof TbelAlarmConditionExpression tbelExpression) { + initTbelExpression(tbelExpression.getExpression()); + } + }); + initialized = true; } } } - public void stop() { - if (calculatedFieldScriptEngine != null) { - calculatedFieldScriptEngine.destroy(); + public double evaluateSimpleExpression(String expressionStr, CalculatedFieldState state) { + Expression expression = simpleExpressions.get(expressionStr).get(); + for (Map.Entry entry : state.getArguments().entrySet()) { + try { + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); + double value = switch (kvEntry.getDataType()) { + case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); + case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); + case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); + case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); + }; + expression.setVariable(entry.getKey(), value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } + return expression.evaluate(); + } + + public ListenableFuture evaluateTbelExpression(String expression, CalculatedFieldState state) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(argNames.size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : argNames) { + var arg = toTbelArgument(argName, state); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments, state.getLatestTimestamp())); + + return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + } + + private TbelCfArg toTbelArgument(String key, CalculatedFieldState state) { + return state.getArguments().get(key).toTbelCfArg(); + } + + private void initTbelExpression(String expression) { + if (tbelExpressions == null) { + tbelExpressions = new HashMap<>(); + } else if (tbelExpressions.containsKey(expression)) { + return; } - if (customExpression != null) { - customExpression.remove(); + CalculatedFieldScriptEngine engine = initEngine(tenantId, expression, tbelInvokeService); + tbelExpressions.put(expression, engine); + } + + private void initSimpleExpression(String expression) { + if (simpleExpressions == null) { + simpleExpressions = new HashMap<>(); + } else if (simpleExpressions.containsKey(expression)) { + return; + } + if (isValidExpression(expression)) { + ThreadLocal compiledExpression = ThreadLocal.withInitial(() -> + ExpressionUtils.createExpression(expression, this.arguments.keySet()) + ); + simpleExpressions.put(expression, compiledExpression); + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); } } @@ -326,21 +402,39 @@ public class CalculatedFieldCtx { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } - public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { - boolean expressionChanged = calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !expression.equals(other.expression); - boolean outputChanged = !output.equals(other.output); - return expressionChanged || outputChanged; + public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { // has changes that do not require state reinit and will be picked up by the state on the fly + if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !expression.equals(other.expression)) { + return true; + } + if (!output.equals(other.output)) { + return true; + } + if (cfType == CalculatedFieldType.ALARM && !calculatedField.getName().equals(other.getCalculatedField().getName())) { + return true; + } + return false; } - public boolean hasStateChanges(CalculatedFieldCtx other) { - boolean typeChanged = !cfType.equals(other.cfType); - boolean argumentsChanged = !arguments.equals(other.arguments); - return typeChanged || argumentsChanged; + public boolean hasStateChanges(CalculatedFieldCtx other) { // has changes that require state reinit (will trigger state.reset() and re-fetch arguments) + boolean hasChanges = !arguments.equals(other.arguments); + if (hasChanges) { + return true; + } + if (cfType == CalculatedFieldType.ALARM) { + var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); + if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || + !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { + hasChanges = true; + } + // TODO: implement rules update logic! + } + return hasChanges; } public boolean hasSchedulingConfigChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration otherConfig) { + && other.calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration otherConfig) { boolean refreshTriggerChanged = thisConfig.isScheduledUpdateEnabled() != otherConfig.isScheduledUpdateEnabled(); boolean refreshIntervalChanged = thisConfig.getScheduledUpdateInterval() != otherConfig.getScheduledUpdateInterval(); return refreshTriggerChanged || refreshIntervalChanged; @@ -348,6 +442,15 @@ public class CalculatedFieldCtx { return false; } + public void stop() { + if (tbelExpressions != null) { + tbelExpressions.values().forEach(CalculatedFieldScriptEngine::destroy); + } + if (simpleExpressions != null) { + simpleExpressions.values().forEach(ThreadLocal::remove); + } + } + public String getSizeExceedsLimitMessage() { return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 5f8e7538c4..b397a8e87d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -17,29 +17,26 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; -import java.util.List; import java.util.Map; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "type" -) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), + @Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), + @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), + @Type(value = AlarmCalculatedFieldState.class, name = "ALARM") }) public interface CalculatedFieldState { @@ -57,11 +54,13 @@ public interface CalculatedFieldState { return false; } - void setRequiredArguments(List requiredArguments); + void init(CalculatedFieldCtx ctx); - boolean updateState(CalculatedFieldCtx ctx, Map argumentValues); + boolean update(CalculatedFieldCtx ctx, Map arguments); - ListenableFuture performCalculation(EntityId entityId, CalculatedFieldCtx ctx); + void reset(CalculatedFieldCtx ctx); + + ListenableFuture performCalculation(CalculatedFieldCtx ctx); @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index fe7dfa04d0..13eaa69ca7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,35 +15,24 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfCtx; -import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -@Data @Slf4j -@NoArgsConstructor @EqualsAndHashCode(callSuper = true) public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { - public ScriptCalculatedFieldState(List requiredArguments) { - super(requiredArguments); + public ScriptCalculatedFieldState(EntityId entityId) { + super(entityId); } @Override @@ -52,30 +41,17 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(EntityId entityId, CalculatedFieldCtx ctx) { - Map arguments = new LinkedHashMap<>(); - List args = new ArrayList<>(ctx.getArgNames().size() + 1); - args.add(new Object()); // first element is a ctx, but we will set it later; - for (String argName : ctx.getArgNames()) { - var arg = toTbelArgument(argName); - arguments.put(argName, arg); - if (arg instanceof TbelCfSingleValueArg svArg) { - args.add(svArg.getValue()); - } else { - args.add(arg); - } - } - args.set(0, new TbelCfCtx(arguments, getLatestTimestamp())); - ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + ListenableFuture resultFuture = ctx.evaluateTbelExpression(ctx.getExpression(), this); Output output = ctx.getOutput(); return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + result -> TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(JacksonUtil.valueToTree(result)) + .build(), MoreExecutors.directExecutor() ); } - private TbelCfArg toTbelArgument(String key) { - return arguments.get(key).toTbelCfArg(); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 80b650fc7c..13aa3fe6fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -19,27 +19,20 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; -import java.util.List; -import java.util.Map; - -@Data -@NoArgsConstructor @EqualsAndHashCode(callSuper = true) public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { - public SimpleCalculatedFieldState(List requiredArguments) { - super(requiredArguments); + public SimpleCalculatedFieldState(EntityId entityId) { + super(entityId); } @Override @@ -55,31 +48,18 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(EntityId entityId, CalculatedFieldCtx ctx) { - var expr = ctx.getCustomExpression().get(); - - for (Map.Entry entry : this.arguments.entrySet()) { - try { - BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); - double value = switch (kvEntry.getDataType()) { - case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); - case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); - case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); - case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); - }; - expr.setVariable(entry.getKey(), value); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); - } - } - - double expressionResult = expr.evaluate(); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + double expressionResult = ctx.evaluateSimpleExpression(ctx.getExpression(), this); Output output = ctx.getOutput(); Object result = formatResult(expressionResult, output.getDecimalsByDefault()); JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), outputResult)); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(outputResult) + .build()); } private Object formatResult(double expressionResult, Integer decimals) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java new file mode 100644 index 0000000000..747846f655 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -0,0 +1,320 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.alarm; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.action.TbAlarmResult; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; +import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.AlarmCalculatedFieldResult; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Function; + +@EqualsAndHashCode(callSuper = true) +@Slf4j +public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { + + private String alarmType; + private AlarmCalculatedFieldConfiguration configuration; + + @Getter + private final Map createRuleStates = new TreeMap<>(Comparator.comparing(Enum::ordinal)); + @Getter + private AlarmRuleState clearRuleState; + + @Getter + private Alarm currentAlarm; + private boolean initialFetchDone; + + public AlarmCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void init(CalculatedFieldCtx ctx) { + super.init(ctx); + + this.alarmType = ctx.getCalculatedField().getName(); + this.configuration = getConfiguration(ctx); + + Map createRules = configuration.getCreateRules(); + createRules.forEach((severity, rule) -> { + AlarmRuleState ruleState = createRuleStates.get(severity); + if (ruleState == null) { + ruleState = new AlarmRuleState(severity, rule, this); + createRuleStates.put(severity, ruleState); + } else { // can be null if was restored + ruleState.setAlarmRule(rule); + // todo: is it enough to just set new alarm rule to alarm rule state? is it ok to leave the state as were?? + } + }); + createRuleStates.keySet().removeIf(severity -> !createRules.containsKey(severity)); + + AlarmRule clearRule = configuration.getClearRule(); + if (clearRule != null) { + if (clearRuleState == null) { + clearRuleState = new AlarmRuleState(null, clearRule, this); + } else { + clearRuleState.setAlarmRule(clearRule); + } + } else { + clearRuleState = null; + } + log.debug("Initialized create rule states {} and clear rule state {} for {}", createRuleStates, clearRuleState, ctx.getCalculatedField()); + } + + @Override + public void reset(CalculatedFieldCtx ctx) { + super.reset(ctx); + } + + @Override + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + initCurrentAlarm(ctx); + AlarmCalculatedFieldResult result = createOrClearAlarms(state -> state.eval(ctx), ctx); + return Futures.immediateFuture(result); + } + + // TODO: harvesting + public ListenableFuture performCalculation(long ts, CalculatedFieldCtx ctx) { + initCurrentAlarm(ctx); + AlarmCalculatedFieldResult result = createOrClearAlarms(ruleState -> ruleState.eval(ts), ctx); + return Futures.immediateFuture(result); + } + + @SneakyThrows + public boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { + if (expression instanceof TbelAlarmConditionExpression tbelExpression) { + Object result = ctx.evaluateTbelExpression(tbelExpression.getExpression(), this).get(); + if (result instanceof Boolean booleanResult) { + return booleanResult; + } else { + throw new IllegalStateException("Condition expression returned non-boolean value: '" + result + "'"); + } + } else { + throw new UnsupportedOperationException("Simple expressions not supported"); + } + } + + public void processAlarmAction(Alarm alarm, ActionType action) { + switch (action) { + case ALARM_ACK -> processAlarmAck(alarm); + case ALARM_CLEAR -> processAlarmClear(alarm); + case ALARM_DELETE -> processAlarmDelete(alarm); + } + } + + private void processAlarmClear(Alarm alarm) { + currentAlarm = null; + createRuleStates.values().forEach(AlarmRuleState::clear); + } + + private void processAlarmAck(Alarm alarm) { + currentAlarm.setAcknowledged(alarm.isAcknowledged()); + currentAlarm.setAckTs(alarm.getAckTs()); + } + + private void processAlarmDelete(Alarm alarm) { + currentAlarm = null; + createRuleStates.values().forEach(AlarmRuleState::clear); + } + + public AlarmCalculatedFieldResult createOrClearAlarms(Function evalFunction, CalculatedFieldCtx ctx) { + TbAlarmResult result = null; + AlarmRuleState resultState = null; + for (AlarmRuleState state : createRuleStates.values()) { + AlarmEvalResult evalResult = evalFunction.apply(state); + log.debug("Evaluated create rule {} with args {}. Result: {}", state, arguments, evalResult); + if (AlarmEvalResult.TRUE.equals(evalResult)) { + resultState = state; + break; + } else if (AlarmEvalResult.FALSE.equals(evalResult)) { + clearAlarmState(state); + } + } + + if (resultState != null) { + result = calculateAlarmResult(resultState, ctx); + log.debug("Alarm result for state {}: {}", resultState, result); + clearAlarmState(clearRuleState); + } else if (currentAlarm != null && clearRuleState != null) { + AlarmEvalResult evalResult = evalFunction.apply(clearRuleState); + log.debug("Evaluated clear rule {} with args {}. Result: {}", clearRuleState, arguments, evalResult); + if (AlarmEvalResult.TRUE.equals(evalResult)) { + clearAlarmState(clearRuleState); + for (AlarmRuleState state : createRuleStates.values()) { + clearAlarmState(state); + } + AlarmApiCallResult clearResult = ctx.getAlarmService().clearAlarm( + ctx.getTenantId(), currentAlarm.getId(), System.currentTimeMillis(), createDetails(clearRuleState), true + ); + if (clearResult.isCleared()) { + result = new TbAlarmResult(false, false, true, clearResult.getAlarm()); + resultState = clearRuleState; + } + currentAlarm = null; + } else if (AlarmEvalResult.FALSE.equals(evalResult)) { + clearAlarmState(clearRuleState); + } + } + return AlarmCalculatedFieldResult.builder() + .alarmResult(result) + .alarmRuleState(resultState) + .build(); + } + + private void clearAlarmState(AlarmRuleState state) { + if (state != null) { + state.clear(); + } + } + + private void initCurrentAlarm(CalculatedFieldCtx ctx) { + if (!initialFetchDone) { + Alarm alarm = ctx.getAlarmService().findLatestActiveByOriginatorAndType(ctx.getTenantId(), entityId, alarmType); + if (alarm != null && !alarm.getStatus().isCleared()) { + currentAlarm = alarm; + } + initialFetchDone = true; + } + } + + private TbAlarmResult calculateAlarmResult(AlarmRuleState ruleState, CalculatedFieldCtx ctx) { + AlarmSeverity severity = ruleState.getSeverity(); + if (currentAlarm != null) { + // TODO: In some extremely rare cases, we might miss the event of alarm clear (If one use in-mem queue and restarted the server) or (if one manipulated the rule chain). + // Maybe we should fetch alarm every time? + currentAlarm.setEndTs(System.currentTimeMillis()); + AlarmSeverity oldSeverity = currentAlarm.getSeverity(); + // Skip update if severity is decreased. + if (severity.ordinal() <= oldSeverity.ordinal()) { + currentAlarm.setDetails(createDetails(ruleState)); + currentAlarm.setSeverity(severity); + AlarmApiCallResult result = ctx.getAlarmService().updateAlarm(AlarmUpdateRequest.fromAlarm(currentAlarm)); + currentAlarm = result.getAlarm(); + return TbAlarmResult.fromAlarmResult(result); + } else { + return null; + } + } else { + var newAlarm = new Alarm(); + newAlarm.setType(alarmType); + newAlarm.setAcknowledged(false); + newAlarm.setCleared(false); + newAlarm.setSeverity(severity); + long startTs = latestTimestamp; + long currentTime = System.currentTimeMillis(); + if (startTs == 0L || startTs > currentTime) { + startTs = currentTime; + } + newAlarm.setStartTs(startTs); + newAlarm.setEndTs(startTs); + newAlarm.setDetails(createDetails(ruleState)); + newAlarm.setOriginator(entityId); + newAlarm.setTenantId(ctx.getTenantId()); + newAlarm.setPropagate(configuration.isPropagate()); + newAlarm.setPropagateToOwner(configuration.isPropagateToOwner()); + newAlarm.setPropagateToTenant(configuration.isPropagateToTenant()); + if (configuration.getPropagateRelationTypes() != null) { + newAlarm.setPropagateRelationTypes(configuration.getPropagateRelationTypes()); + } + AlarmApiCallResult result = ctx.getAlarmService().createAlarm(AlarmCreateOrUpdateActiveRequest.fromAlarm(newAlarm)); + currentAlarm = result.getAlarm(); + return TbAlarmResult.fromAlarmResult(result); + } + } + + private JsonNode createDetails(AlarmRuleState ruleState) { + JsonNode alarmDetails; + String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails(); + DashboardId dashboardId = ruleState.getAlarmRule().getDashboardId(); + + if (StringUtils.isNotEmpty(alarmDetailsStr) || dashboardId != null) { + ObjectNode newDetails = JacksonUtil.newObjectNode(); + if (StringUtils.isNotEmpty(alarmDetailsStr)) { + for (Map.Entry entry : arguments.entrySet()) { + String key = entry.getKey(); + ArgumentEntry value = entry.getValue(); + alarmDetailsStr = alarmDetailsStr.replaceAll(String.format("\\$\\{%s}", key), String.valueOf(value.getValue())); + } + newDetails.put("data", alarmDetailsStr); + } + if (dashboardId != null) { + newDetails.put("dashboardId", dashboardId.getId().toString()); + } + alarmDetails = newDetails; + } else if (currentAlarm != null) { + alarmDetails = currentAlarm.getDetails(); + } else { + alarmDetails = JacksonUtil.newObjectNode(); + } + + return alarmDetails; + } + + protected SingleValueArgumentEntry getArgument(String key) { + SingleValueArgumentEntry entry = (SingleValueArgumentEntry) arguments.get(key); + if (entry == null) { + throw new IllegalArgumentException("Argument '" + key + "' is missing"); + } + return entry; + } + + private AlarmCalculatedFieldConfiguration getConfiguration(CalculatedFieldCtx ctx) { + return (AlarmCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + } + + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (!(newEntry instanceof SingleValueArgumentEntry)) { + throw new IllegalArgumentException("Only single value arguments supported"); + } + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.ALARM; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmStateUpdateResult.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java similarity index 82% rename from rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmStateUpdateResult.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java index ba7dc5dce8..6775b14586 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmStateUpdateResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.profile; +package org.thingsboard.server.service.cf.ctx.state.alarm; -enum AlarmStateUpdateResult { +public enum AlarmEvalResult { - NONE, CREATED, UPDATED, SEVERITY_UPDATED, CLEARED; + FALSE, NOT_YET_TRUE, TRUE; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java new file mode 100644 index 0000000000..04386c68f6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -0,0 +1,233 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.alarm; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.KvUtil; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; +import org.thingsboard.server.common.data.alarm.rule.condition.DurationAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.RepeatingAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeSchedule; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeScheduleItem; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.SpecificTimeSchedule; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.msg.tools.SchedulerUtils; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Optional; +import java.util.function.Function; + +@Data +@Slf4j +public class AlarmRuleState { + + private final AlarmSeverity severity; + private AlarmRule alarmRule; + private AlarmCalculatedFieldState state; + + private AlarmCondition condition; + + private long lastEventTs; + private long duration; + private long eventCount; + + public AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, AlarmCalculatedFieldState state) { + this.severity = severity; + if (alarmRule != null) { + setAlarmRule(alarmRule); + } + this.state = state; + } + + public AlarmEvalResult eval(CalculatedFieldCtx ctx) { + boolean active = isActive(state.getLatestTimestamp()); + return switch (condition.getType()) { + case SIMPLE -> (active && eval(condition.getExpression(), ctx)) ? AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; + case DURATION -> evalDuration(active, ctx); + case REPEATING -> evalRepeating(active, ctx); + }; + } + + public AlarmEvalResult eval(long ts) { + switch (condition.getType()) { + case SIMPLE: + case REPEATING: + return AlarmEvalResult.NOT_YET_TRUE; + case DURATION: + long requiredDurationInMs = getRequiredDurationInMs(); + if (requiredDurationInMs > 0 && lastEventTs > 0 && ts > lastEventTs) { + long duration = this.duration + (ts - lastEventTs); + if (isActive(ts)) { + return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; + } + } + default: + return AlarmEvalResult.FALSE; + } + } + + private boolean isActive(long eventTs) { + if (condition.getSchedule() == null) { + return true; + } + AlarmSchedule schedule = getValue(condition.getSchedule(), entry -> Optional.ofNullable(KvUtil.getStringValue(entry)) + .map(str -> JsonConverter.parse(str, AlarmSchedule.class)) + .orElse(null)); + return switch (schedule.getType()) { + case ANY_TIME -> true; + case SPECIFIC_TIME -> isActiveSpecific((SpecificTimeSchedule) schedule, eventTs); + case CUSTOM -> isActiveCustom((CustomTimeSchedule) schedule, eventTs); + }; + } + + private boolean isActiveSpecific(SpecificTimeSchedule schedule, long eventTs) { + ZoneId zoneId = SchedulerUtils.getZoneId(schedule.getTimezone()); + ZonedDateTime zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(eventTs), zoneId); + if (schedule.getDaysOfWeek().size() != 7) { + int dayOfWeek = zdt.getDayOfWeek().getValue(); + if (!schedule.getDaysOfWeek().contains(dayOfWeek)) { + return false; + } + } + long endsOn = schedule.getEndsOn(); + if (endsOn == 0) { + // 24 hours in milliseconds + endsOn = 86400000; + } + + return isActive(eventTs, zoneId, zdt, schedule.getStartsOn(), endsOn); + } + + private boolean isActiveCustom(CustomTimeSchedule schedule, long eventTs) { + ZoneId zoneId = SchedulerUtils.getZoneId(schedule.getTimezone()); + ZonedDateTime zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(eventTs), zoneId); + int dayOfWeek = zdt.toLocalDate().getDayOfWeek().getValue(); + for (CustomTimeScheduleItem item : schedule.getItems()) { + if (item.getDayOfWeek() == dayOfWeek) { + if (item.isEnabled()) { + long endsOn = item.getEndsOn(); + if (endsOn == 0) { + // 24 hours in milliseconds + endsOn = 86400000; + } + return isActive(eventTs, zoneId, zdt, item.getStartsOn(), endsOn); + } else { + return false; + } + } + } + return false; + } + + private boolean isActive(long eventTs, ZoneId zoneId, ZonedDateTime zdt, long startsOn, long endsOn) { + long startOfDay = zdt.toLocalDate().atStartOfDay(zoneId).toInstant().toEpochMilli(); + long msFromStartOfDay = eventTs - startOfDay; + if (startsOn <= endsOn) { + return startsOn <= msFromStartOfDay && endsOn > msFromStartOfDay; + } else { + return startsOn < msFromStartOfDay || (0 < msFromStartOfDay && msFromStartOfDay < endsOn); + } + } + + public void clear() { + eventCount = 0L; + lastEventTs = 0L; + duration = 0L; + } + + private AlarmEvalResult evalRepeating(boolean active, CalculatedFieldCtx ctx) { + if (active && eval(condition.getExpression(), ctx)) { + eventCount++; + long requiredRepeats = getIntValue(((RepeatingAlarmCondition) condition).getCount()); + return eventCount >= requiredRepeats ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; + } + } + + private AlarmEvalResult evalDuration(boolean active, CalculatedFieldCtx ctx) { + if (active && eval(condition.getExpression(), ctx)) { + if (lastEventTs > 0) { + if (state.getLatestTimestamp() > lastEventTs) { + duration = duration + (state.getLatestTimestamp() - lastEventTs); + lastEventTs = state.getLatestTimestamp(); + } + } else { + lastEventTs = state.getLatestTimestamp(); + duration = 0L; + } + long requiredDurationInMs = getRequiredDurationInMs(); + return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; + } + } + + private Integer getIntValue(AlarmConditionValue value) { + return getValue(value, entry -> Optional.ofNullable(KvUtil.getLongValue(entry)).map(Long::intValue).orElse(null)); + } + + private long getRequiredDurationInMs() { + return getValue(((DurationAlarmCondition) condition).getValue(), KvUtil::getLongValue); + } + + private boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { + return state.eval(expression, ctx); + } + + private T getValue(AlarmConditionValue conditionValue, Function mapper) { + T value = conditionValue.getStaticValue(); + if (value == null) { + String argument = conditionValue.getDynamicValueArgument(); + SingleValueArgumentEntry entry = state.getArgument(argument); + value = mapper.apply(entry.getKvEntryValue()); + if (value == null) { + throw new IllegalArgumentException("No value found for argument " + argument); + } + } + return value; + } + + public void setAlarmRule(AlarmRule alarmRule) { + this.alarmRule = alarmRule; + this.condition = alarmRule.getCondition(); + } + + @Override + public String toString() { + return "AlarmRuleState{" + + "severity=" + severity + + ", condition=" + condition + + ", lastEventTs=" + lastEventTs + + ", duration=" + duration + + ", eventCount=" + eventCount + + '}'; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java index 506ddcff78..b418dc73d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java @@ -19,8 +19,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.geo.Coordinates; @@ -32,6 +33,7 @@ import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupC import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; @@ -39,7 +41,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -49,20 +50,16 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.INSIDE; import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.OUTSIDE; -@Data +@Getter +@Setter @Slf4j @EqualsAndHashCode(callSuper = true) public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { - private boolean dirty; + private boolean dirty = false; - public GeofencingCalculatedFieldState() { - super(new ArrayList<>(), new HashMap<>(), false, -1); - this.dirty = false; - } - - public GeofencingCalculatedFieldState(List argNames) { - super(argNames); + public GeofencingCalculatedFieldState(EntityId entityId) { + super(entityId); } @Override @@ -71,11 +68,7 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public boolean updateState(CalculatedFieldCtx ctx, Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - + public boolean update(CalculatedFieldCtx ctx, Map argumentValues) { boolean stateUpdated = false; for (var entry : argumentValues.entrySet()) { @@ -117,7 +110,7 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(EntityId entityId, CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { double latitude = (double) arguments.get(ENTITY_ID_LATITUDE_ARGUMENT_KEY).getValue(); double longitude = (double) arguments.get(ENTITY_ID_LONGITUDE_ARGUMENT_KEY).getValue(); Coordinates entityCoordinates = new Coordinates(latitude, longitude); @@ -157,13 +150,23 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { updateResultNode(argumentKey, zoneResults, zoneGroupCfg.getReportStrategy(), resultNode); }); - var result = new CalculatedFieldResult(ctx.getOutput().getType(), ctx.getOutput().getScope(), resultNode); + var result = TelemetryCalculatedFieldResult.builder() + .type(ctx.getOutput().getType()) + .scope(ctx.getOutput().getScope()) + .result(resultNode) + .build(); if (relationFutures.isEmpty()) { return Futures.immediateFuture(result); } return Futures.whenAllComplete(relationFutures).call(() -> result, MoreExecutors.directExecutor()); } + @Override + public void reset(CalculatedFieldCtx ctx) { + super.reset(ctx); + dirty = false; + } + private Map getGeofencingArguments() { return arguments.entrySet() .stream() diff --git a/application/src/main/java/org/thingsboard/server/service/edge/RelatedEdgesSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/RelatedEdgesSourcingListener.java index 8a111e4d9d..d942dc2277 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/RelatedEdgesSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/RelatedEdgesSourcingListener.java @@ -54,16 +54,18 @@ public class RelatedEdgesSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(ActionEntityEvent event) { - executorService.submit(() -> { - log.trace("[{}] ActionEntityEvent called: {}", event.getTenantId(), event); - try { - switch (event.getActionType()) { - case ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE -> relatedEdgesService.publishRelatedEdgeIdsEvictEvent(event.getTenantId(), event.getEntityId()); - } - } catch (Exception e) { - log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event, e); + switch (event.getActionType()) { + case ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE -> { + executorService.submit(() -> { + log.trace("[{}] ActionEntityEvent called: {}", event.getTenantId(), event); + try { + relatedEdgesService.publishRelatedEdgeIdsEvictEvent(event.getTenantId(), event.getEntityId()); + } catch (Exception e) { + log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event, e); + } + }); } - }); + } } @TransactionalEventListener( diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java index e8c2f65975..d6fd2f6968 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java @@ -77,7 +77,7 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { case ALARM_CLEAR_RPC_MESSAGE: Alarm alarmToClear = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); if (alarmToClear != null) { - edgeCtx.getAlarmService().clearAlarm(tenantId, alarmId, alarm.getClearTs(), alarm.getDetails()); + edgeCtx.getAlarmService().clearAlarm(tenantId, alarmId, alarm.getClearTs(), alarm.getDetails(), true); } break; case ENTITY_DELETED_RPC_MESSAGE: diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 03ab77ac09..aa308919db 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.JobManager; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Device; @@ -31,9 +32,11 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -53,13 +56,17 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.rule.engine.api.JobManager; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import java.util.Set; @@ -72,6 +79,7 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final EdgeSynchronizationManager edgeSynchronizationManager; private final JobManager jobManager; + private final CalculatedFieldCache calculatedFieldCache; @PostConstruct public void init() { @@ -153,7 +161,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (entityType != EntityType.TENANT && !tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -216,18 +224,46 @@ public class EntityStateSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(ActionEntityEvent event) { - log.trace("[{}] ActionEntityEvent called: {}", event.getTenantId(), event); - if (ActionType.CREDENTIALS_UPDATED.equals(event.getActionType()) && - EntityType.DEVICE.equals(event.getEntityId().getEntityType()) - && event.getEntity() instanceof DeviceCredentials) { - tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(event.getTenantId(), - (DeviceId) event.getEntityId(), (DeviceCredentials) event.getEntity()), null); - } else if (ActionType.ASSIGNED_TO_TENANT.equals(event.getActionType()) && event.getEntity() instanceof Device device) { - Tenant tenant = JacksonUtil.fromString(event.getBody(), Tenant.class); - if (tenant != null) { - tbClusterService.onDeviceAssignedToTenant(tenant.getId(), device); + TenantId tenantId = event.getTenantId(); + log.trace("[{}] ActionEntityEvent called: {}", tenantId, event); + switch (event.getActionType()) { + case CREDENTIALS_UPDATED -> { + if (EntityType.DEVICE.equals(event.getEntityId().getEntityType()) && + event.getEntity() instanceof DeviceCredentials deviceCredentials) { + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(tenantId, + (DeviceId) event.getEntityId(), deviceCredentials), null); + } + } + case ASSIGNED_TO_TENANT -> { + if (event.getEntity() instanceof Device device) { + Tenant tenant = JacksonUtil.fromString(event.getBody(), Tenant.class); + if (tenant != null) { + tbClusterService.onDeviceAssignedToTenant(tenant.getId(), device); + } + pushAssignedFromNotification(tenant, tenantId, device); + } + } + case ALARM_ACK, ALARM_CLEAR, ALARM_DELETE -> { + if (event.getActionType() == ActionType.ALARM_DELETE && !tenantExists(tenantId)) { + return; + } + Alarm alarm = (Alarm) event.getEntity(); + if (calculatedFieldCache.hasCalculatedFields(tenantId, alarm.getOriginator(), ctx -> ctx.getCfType() == CalculatedFieldType.ALARM)) { + ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder() + .setEventMsg(toProto(event)) + .addCfTypes(CalculatedFieldType.ALARM.name()) + .build(); + tbClusterService.pushMsgToCalculatedFields(tenantId, alarm.getOriginator(), msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) {} + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to push alarm event to CF queue: {}", tenantId, event, t); + } + }); + } } - pushAssignedFromNotification(tenant, event.getTenantId(), device); } } @@ -338,6 +374,10 @@ public class EntityStateSourcingListener { } } + private boolean tenantExists(TenantId tenantId) { + return tenantId.isSysTenantId() || tenantService.tenantExists(tenantId); + } + private TbMsgMetaData getMetaDataForAssignedFrom(Tenant tenant) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("assignedFromTenantId", tenant.getId().getId().toString()); @@ -345,4 +385,13 @@ public class EntityStateSourcingListener { return metaData; } + private EntityActionEventProto toProto(ActionEntityEvent event) { + return EntityActionEventProto.newBuilder() + .setTenantId(ProtoUtils.toProto(event.getTenantId())) + .setEntityId(ProtoUtils.toProto(event.getEntityId())) + .setAction(event.getActionType().name()) + .setEntity(event.getEntity() != null ? JacksonUtil.toString(event.getEntity()) : "") + .build(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 8d4ab25578..53be45bc2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PreDestroy; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; @@ -22,10 +23,12 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldEntityActionEventMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -57,6 +60,7 @@ import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedCon import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; +import java.util.EnumSet; import java.util.List; import java.util.Set; import java.util.UUID; @@ -158,12 +162,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa try { ToCalculatedFieldMsg toCfMsg = msg.getValue(); pendingMsgHolder.setMsg(toCfMsg); - if (toCfMsg.hasTelemetryMsg()) { - log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); - forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); - } else if (toCfMsg.hasLinkedTelemetryMsg()) { - forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); - } + processMsg(toCfMsg, id, callback); } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); callback.onFailure(e); @@ -181,6 +180,18 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa consumer.commit(); } + private void processMsg(ToCalculatedFieldMsg toCfMsg, UUID id, TbCallback callback) { + Set cfTypes = getCfTypes(toCfMsg.getCfTypesList()); + if (toCfMsg.hasTelemetryMsg()) { // TODO: add CF type filter to the message. or just rename the CF strategy to "Process alarms and calculated fields + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasEventMsg()) { + actorContext.tell(CalculatedFieldEntityActionEventMsg.fromProto(toCfMsg.getEventMsg(), callback)); + } + } + @Override protected ServiceType getServiceType() { return ServiceType.TB_RULE_ENGINE; @@ -251,6 +262,18 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); } + private Set getCfTypes(ProtocolStringList cfTypesList) { + Set cfTypes; + if (cfTypesList.isEmpty()) { + cfTypes = EnumSet.allOf(CalculatedFieldType.class); + } else { + cfTypes = cfTypesList.stream() + .map(CalculatedFieldType::valueOf) + .collect(Collectors.toSet()); + } + return cfTypes; + } + @Override protected void stopConsumers() { super.stopConsumers(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java index 8c4a375fae..b68f604460 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java @@ -102,7 +102,12 @@ public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService @Override public AlarmApiCallResult clearAlarm(TenantId tenantId, AlarmId alarmId, long clearTs, JsonNode details) { - return withWsCallback(alarmService.clearAlarm(tenantId, alarmId, clearTs, details)); + return clearAlarm(tenantId, alarmId, clearTs, details, true); + } + + @Override + public AlarmApiCallResult clearAlarm(TenantId tenantId, AlarmId alarmId, long clearTs, JsonNode details, boolean pushEvent) { + return withWsCallback(alarmService.clearAlarm(tenantId, alarmId, clearTs, details, pushEvent)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 69b41addf9..cd5848ad0d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -169,6 +169,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(resultFuture, result -> { if (strategy.processCalculatedFields()) { + // TODO: divide CFs and alarm rules processing calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); } else { request.getCallback().onSuccess(null); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 934eadd98f..37700adc00 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.apache.commons.lang3.math.NumberUtils; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; @@ -28,10 +29,11 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import java.util.Optional; @@ -64,11 +66,12 @@ public class CalculatedFieldArgumentUtils { return new StringDataEntry(key, defaultValue); } - public static CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + public static CalculatedFieldState createStateByType(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { - case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); - case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); - case GEOFENCING -> new GeofencingCalculatedFieldState(ctx.getArgNames()); + case SIMPLE -> new SimpleCalculatedFieldState(entityId); + case SCRIPT -> new ScriptCalculatedFieldState(entityId); + case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); + case ALARM -> new AlarmCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 4e93c8233e..38aeb45a20 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -17,6 +17,7 @@ package org.thingsboard.server.utils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -26,6 +27,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -38,13 +41,15 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; +import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; import java.util.Map; import java.util.Optional; @@ -95,9 +100,27 @@ public class CalculatedFieldUtils { builder.addGeofencingArguments(toGeofencingArgumentProto(argName, geofencingArgumentEntry)); } }); + if (state instanceof AlarmCalculatedFieldState alarmState) { + AlarmStateProto.Builder alarmStateProto = AlarmStateProto.newBuilder(); + alarmState.getCreateRuleStates().forEach((severity, ruleState) -> { + alarmStateProto.addCreateRuleStates(toAlarmRuleStateProto(ruleState)); + }); + if (alarmState.getClearRuleState() != null) { + alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); + } + } return builder.build(); } + private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) { + return AlarmRuleStateProto.newBuilder() + .setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse("")) + .setLastEventTs(ruleState.getLastEventTs()) + .setDuration(ruleState.getDuration()) + .setEventCount(ruleState.getEventCount()) + .build(); + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -143,7 +166,7 @@ public class CalculatedFieldUtils { return builder.build(); } - public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + public static CalculatedFieldState fromProto(CalculatedFieldEntityCtxId id, CalculatedFieldStateProto proto) { if (StringUtils.isEmpty(proto.getType())) { return null; } @@ -151,22 +174,36 @@ public class CalculatedFieldUtils { CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); - case GEOFENCING -> new GeofencingCalculatedFieldState(); + case SIMPLE -> new SimpleCalculatedFieldState(id.entityId()); + case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); + case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); + case ALARM -> new AlarmCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - if (CalculatedFieldType.SCRIPT.equals(type)) { - proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); - } - - if (CalculatedFieldType.GEOFENCING.equals(type)) { - proto.getGeofencingArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto))); + switch (type) { + case SCRIPT -> { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + case GEOFENCING -> { + proto.getGeofencingArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto))); + } + case ALARM -> { + AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state; + AlarmStateProto alarmStateProto = proto.getAlarmState(); + for (AlarmRuleStateProto ruleStateProto : alarmStateProto.getCreateRuleStatesList()) { + AlarmSeverity severity = StringUtils.isNotEmpty(ruleStateProto.getSeverity()) ? AlarmSeverity.valueOf(ruleStateProto.getSeverity()) : null; + AlarmRuleState ruleState = new AlarmRuleState(severity, null, alarmState); + ruleState.setLastEventTs(ruleStateProto.getLastEventTs()); + ruleState.setDuration(ruleStateProto.getDuration()); + ruleState.setEventCount(ruleStateProto.getEventCount()); + alarmState.getCreateRuleStates().put(severity, ruleState); + } + } } return state; diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java new file mode 100644 index 0000000000..5cf674f6b6 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -0,0 +1,191 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.action.TbAlarmResult; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.SimpleAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.event.EventDao; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +@DaoSqlTest +public class AlarmRulesTest extends AbstractControllerTest { + + @MockitoSpyBean + private ActorSystemContext actorSystemContext; + + @Autowired + private EventDao eventDao; + + private DeviceId deviceId; + private EventId latestEventId; + + @Before + public void beforeEach() throws Exception { + loginTenantAdmin(); + Device device = createDevice("Device A", "aaa"); + deviceId = device.getId(); + } + + @Test + public void testCreateAndSeverityUpdateAndClear() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + Map createRules = Map.of( + AlarmSeverity.MAJOR, "return temperature >= 50;", + AlarmSeverity.CRITICAL, "return temperature >= 100;" + ); + String clearRule = "return temperature <= 25;"; + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, clearRule); + + postTelemetry(deviceId, "{\"temperature\":50}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + + postTelemetry(deviceId, "{\"temperature\":100}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isSeverityUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + + postTelemetry(deviceId, "{\"temperature\":101}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + + postTelemetry(deviceId, "{\"temperature\":20}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCleared()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.CLEARED_UNACK); + }); + } + + private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + TbAlarmResult alarmResult = getLatestAlarmResult(calculatedField.getId()); + assertThat(alarmResult).isNotNull(); + assertion.accept(alarmResult); + + Alarm alarm = alarmResult.getAlarm(); + assertThat(alarm.getOriginator()).isEqualTo(deviceId); + assertThat(alarm.getType()).isEqualTo(calculatedField.getName()); + }); + } + + private TbAlarmResult getLatestAlarmResult(CalculatedFieldId calculatedFieldId) { + List debugEvents = getDebugEvents(calculatedFieldId, 1); + if (debugEvents.isEmpty()) { + return null; + } + CalculatedFieldDebugEvent debugEvent = debugEvents.get(0); + if (debugEvent.getError() != null) { + System.err.println("CF error: " + debugEvent.getError()); + Assertions.fail(); + } + if (debugEvent.getId().equals(latestEventId)) { + return null; + } + latestEventId = debugEvent.getId(); + return JacksonUtil.fromString(debugEvent.getResult(), TbAlarmResult.class); + } + + private CalculatedField createAlarmCf(EntityId entityId, + String alarmType, + Map arguments, + Map createConditions, + String clearCondition) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(entityId); + calculatedField.setName(alarmType); + calculatedField.setType(CalculatedFieldType.ALARM); + AlarmCalculatedFieldConfiguration configuration = new AlarmCalculatedFieldConfiguration(); + configuration.setArguments(arguments); + configuration.setCreateRules(new HashMap<>()); + createConditions.forEach((severity, expression) -> { + configuration.getCreateRules().put(severity, toAlarmRule(expression)); + }); + configuration.setClearRule(toAlarmRule(clearCondition)); + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private AlarmRule toAlarmRule(String conditionExpression) { + if (conditionExpression == null) { + return null; + } + AlarmRule rule = new AlarmRule(); + SimpleAlarmCondition condition = new SimpleAlarmCondition(); + TbelAlarmConditionExpression expression = new TbelAlarmConditionExpression(); + expression.setExpression(conditionExpression); + condition.setExpression(expression); + rule.setCondition(condition); + return rule; + } + + private List getDebugEvents(CalculatedFieldId calculatedFieldId, int limit) { + return eventDao.findLatestEvents(tenantId.getId(), calculatedFieldId.getId(), EventType.DEBUG_CALCULATED_FIELD, limit).stream() + .map(e -> (CalculatedFieldDebugEvent) e).toList(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index b6a1faa1ed..28e9d5bb1a 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -75,7 +75,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes @Test public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + postTelemetry(testDevice.getId(), "{\"temperature\":25}"); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); CalculatedField calculatedField = new CalculatedField(); @@ -112,7 +112,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); }); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + postTelemetry(testDevice.getId(), "{\"temperature\":30}"); await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -133,6 +133,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0)).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); }); @@ -197,7 +198,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); }); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + postTelemetry(testDevice.getId(), "{\"temperature\":30}"); await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -246,7 +247,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); }); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + postTelemetry(testDevice.getId(), "{\"temperature\":30}"); await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -431,7 +432,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes @Test public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + postTelemetry(testDevice.getId(), "{\"temperature\":25}"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -467,7 +468,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); }); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + postTelemetry(testDevice.getId(), "{\"temperature\":30}"); await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -482,7 +483,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes public void testSimpleCalculatedFieldWhenUseLatestTsIsTrue() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); + postTelemetry(testDevice.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -526,10 +527,10 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes long ts = System.currentTimeMillis(); long tsA = ts - 300000L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":1}}", tsA))); + postTelemetry(testDevice.getId(), String.format("{\"ts\": %s, \"values\": {\"a\":1}}", tsA)); long tsB = ts - 300L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":5}}", tsB))); + postTelemetry(testDevice.getId(), String.format("{\"ts\": %s, \"values\": {\"b\":5}}", tsB)); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -570,7 +571,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); long tsABeforeTsB = tsB - 300L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":10}}", tsABeforeTsB))); + postTelemetry(testDevice.getId(), String.format("{\"ts\": %s, \"values\": {\"a\":10}}", tsABeforeTsB)); await().alias("update telemetry with ts less than latest -> save result with latest ts").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -586,7 +587,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes public void testScriptCalculatedFieldWhenUsedLatestTsInScript() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); + postTelemetry(testDevice.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index cf9d2feb23..89b2681015 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.controller; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EventInfo; -import org.thingsboard.server.common.data.event.EventType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; @@ -61,18 +55,6 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle return doGet("/api/ruleChain/metadata/" + ruleChainId.getId().toString(), RuleChainMetaData.class); } - protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { - return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE.getOldName(), limit); - } - - protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { - TimePageLink pageLink = new TimePageLink(limit); - return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&", - new TypeReference>() { - }, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId()); - } - - protected JsonNode getMetadata(EventInfo outEvent) { String metaDataStr = outEvent.getBody().get("metadata").asText(); try { diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index fd01581e36..8e7d2bb5ff 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -78,10 +78,12 @@ import org.thingsboard.server.actors.device.SessionInfo; import org.thingsboard.server.actors.device.ToDeviceRpcRequestMetadata; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResourceInfo; @@ -89,6 +91,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -101,6 +104,7 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; @@ -1312,4 +1316,24 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { doPost("/api/job/" + jobId + "/reprocess").andExpect(status().isOk()); } + protected void postTelemetry(EntityId entityId, String payload) throws Exception { + doPost("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload)).andExpect(status().isOk()); + } + + protected CalculatedField saveCalculatedField(CalculatedField calculatedField) { + return doPost("/api/calculatedField", calculatedField, CalculatedField.class); + } + + protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { + return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE, limit); + } + + protected PageData getEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit) throws Exception { + TimePageLink pageLink = new TimePageLink(limit); + return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&", + new TypeReference>() { + }, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId()); + } + } diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 3e29f212f0..fb99d6ad0e 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -118,7 +118,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), EventType.LC_EVENT.getOldName(), 1000) + List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), EventType.LC_EVENT, 1000) .getData().stream().filter(e -> { var body = e.getBody(); return body.has("event") && body.get("event").asText().equals("STARTED") diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index 691a1f7ec4..bfc5bd36e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -20,9 +20,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -43,7 +45,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -91,13 +93,15 @@ public class GeofencingCalculatedFieldStateTest { private ApiLimitService apiLimitService; @Mock private RelationService relationService; + @InjectMocks + private ActorSystemContext systemContext; @BeforeEach void setUp() { when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); - ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService, relationService); + ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); - state = new GeofencingCalculatedFieldState(ctx.getArgNames()); + state = new GeofencingCalculatedFieldState(ctx.getEntityId()); } @Test @@ -113,7 +117,7 @@ public class GeofencingCalculatedFieldStateTest { )); Map newArgs = Map.of("allowedZones", geofencingAllowedZoneArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -127,21 +131,21 @@ public class GeofencingCalculatedFieldStateTest { @Test void testUpdateStateWithInvalidArgumentTypeForLatitudeArgument() { - assertThatThrownBy(() -> state.updateState(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for latitude argument: GEOFENCING. Only SINGLE_VALUE type is allowed."); } @Test void testUpdateStateWithInvalidArgumentTypeForLongitudeArgument() { - assertThatThrownBy(() -> state.updateState(ctx, Map.of(ENTITY_ID_LONGITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LONGITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for longitude argument: GEOFENCING. Only SINGLE_VALUE type is allowed."); } @Test void testUpdateStateWithInvalidArgumentTypeForGeofencingArgument() { - assertThatThrownBy(() -> state.updateState(ctx, Map.of("someArgumentName", latitudeArgEntry))) + assertThatThrownBy(() -> state.update(ctx, Map.of("someArgumentName", latitudeArgEntry))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for someArgumentName argument: SINGLE_VALUE. Only GEOFENCING type is allowed."); } @@ -152,7 +156,7 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 50.4760), 190L); Map newArgs = Map.of("latitude", newArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).isEqualTo(newArgs); @@ -164,7 +168,7 @@ public class GeofencingCalculatedFieldStateTest { Map newArgs = Map.of("allowedZones", geofencingAllowedZoneArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isFalse(); assertThat(state.getArguments()).isEqualTo(newArgs); @@ -174,7 +178,7 @@ public class GeofencingCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingSingleValueArgumentEntryWithValueOfAnotherType() { state.arguments = new HashMap<>(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, latitudeArgEntry)); - assertThatThrownBy(() -> state.updateState(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for single value argument entry: GEOFENCING"); } @@ -184,7 +188,7 @@ public class GeofencingCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingGeofencingValueArgumentEntryWithValueOfAnotherType() { state.arguments = new HashMap<>(Map.of("allowedZones", geofencingAllowedZoneArgEntry)); - assertThatThrownBy(() -> state.updateState(ctx, Map.of("allowedZones", latitudeArgEntry))) + assertThatThrownBy(() -> state.update(ctx, Map.of("allowedZones", latitudeArgEntry))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for geofencing argument entry: SINGLE_VALUE"); } @@ -234,7 +238,7 @@ public class GeofencingCalculatedFieldStateTest { when(relationService.saveRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); when(relationService.deleteRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); assertThat(result.getType()).isEqualTo(output.getType()); @@ -250,9 +254,9 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.updateState(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); - CalculatedFieldResult result2 = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result2 = performCalculation(); assertThat(result2).isNotNull(); assertThat(result2.getType()).isEqualTo(output.getType()); @@ -309,7 +313,7 @@ public class GeofencingCalculatedFieldStateTest { when(relationService.saveRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); when(relationService.deleteRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); assertThat(result.getType()).isEqualTo(output.getType()); @@ -322,9 +326,9 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.updateState(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); - CalculatedFieldResult result2 = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result2 = performCalculation(); assertThat(result2).isNotNull(); assertThat(result2.getType()).isEqualTo(output.getType()); @@ -379,7 +383,7 @@ public class GeofencingCalculatedFieldStateTest { when(relationService.saveRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); when(relationService.deleteRelationAsync(any(), any())).thenReturn(Futures.immediateFuture(true)); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); assertThat(result.getType()).isEqualTo(output.getType()); @@ -394,9 +398,9 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.updateState(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); - CalculatedFieldResult result2 = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result2 = performCalculation(); assertThat(result2).isNotNull(); assertThat(result2.getType()).isEqualTo(output.getType()); @@ -480,4 +484,8 @@ public class GeofencingCalculatedFieldStateTest { return config; } + private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { + return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 8c714bc0e7..972d83f8a8 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.service.cf.ctx.state; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -41,10 +43,9 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.stats.DefaultStatsFactory; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -77,10 +78,15 @@ public class ScriptCalculatedFieldStateTest { @BeforeEach void setUp() { + ActorSystemContext systemContext = Mockito.mock(ActorSystemContext.class); + when(systemContext.getTbelInvokeService()).thenReturn(tbelInvokeService); + when(systemContext.getApiLimitService()).thenReturn(apiLimitService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); - ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService, null); + ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); - state = new ScriptCalculatedFieldState(ctx.getArgNames()); + state = new ScriptCalculatedFieldState(ctx.getEntityId()); + state.init(ctx); } @Test @@ -93,7 +99,7 @@ public class ScriptCalculatedFieldStateTest { state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -110,7 +116,7 @@ public class ScriptCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -125,7 +131,7 @@ public class ScriptCalculatedFieldStateTest { void testPerformCalculation() throws ExecutionException, InterruptedException { state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); Output output = getCalculatedFieldConfig().getOutput(); @@ -141,7 +147,7 @@ public class ScriptCalculatedFieldStateTest { "assetHumidity", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("a", 45L), 10L) )); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); Output output = getCalculatedFieldConfig().getOutput(); @@ -221,4 +227,8 @@ public class ScriptCalculatedFieldStateTest { return config; } + private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { + return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + } + } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 8c631ecf6f..30b79b0768 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -18,9 +18,11 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -39,7 +41,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import java.util.HashMap; import java.util.Map; @@ -67,13 +69,15 @@ public class SimpleCalculatedFieldStateTest { @Mock private ApiLimitService apiLimitService; + @InjectMocks + private ActorSystemContext systemContext; @BeforeEach void setUp() { when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); - ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService, null); + ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); - state = new SimpleCalculatedFieldState(ctx.getArgNames()); + state = new SimpleCalculatedFieldState(ctx.getEntityId()); } @Test @@ -89,7 +93,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", key3ArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -107,7 +111,7 @@ public class SimpleCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); - boolean stateUpdated = state.updateState(ctx, newArgs); + boolean stateUpdated = state.update(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); @@ -121,7 +125,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); - assertThatThrownBy(() -> state.updateState(ctx, newArgs)) + assertThatThrownBy(() -> state.update(ctx, newArgs)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @@ -134,7 +138,7 @@ public class SimpleCalculatedFieldStateTest { "key3", key3ArgEntry )); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); Output output = getCalculatedFieldConfig().getOutput(); @@ -151,7 +155,7 @@ public class SimpleCalculatedFieldStateTest { "key3", key3ArgEntry )); - assertThatThrownBy(() -> state.performCalculation(ctx.getEntityId(), ctx)) + assertThatThrownBy(() -> state.performCalculation(ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Argument 'key2' is not a number."); } @@ -164,7 +168,7 @@ public class SimpleCalculatedFieldStateTest { "key3", key3ArgEntry )); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); Output output = getCalculatedFieldConfig().getOutput(); @@ -185,7 +189,7 @@ public class SimpleCalculatedFieldStateTest { output.setDecimalsByDefault(3); ctx.setOutput(output); - CalculatedFieldResult result = state.performCalculation(ctx.getEntityId(), ctx).get(); + TelemetryCalculatedFieldResult result = performCalculation(); assertThat(result).isNotNull(); assertThat(result.getType()).isEqualTo(output.getType()); @@ -265,4 +269,8 @@ public class SimpleCalculatedFieldStateTest { return config; } + private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { + return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + } + } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java index 2697b2b804..acdf7bbf36 100644 --- a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java @@ -36,7 +36,6 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -85,14 +84,14 @@ class CalculatedFieldUtilsTest { geofencingArgumentEntry.setZoneStates(zoneStates); // Create cf state with the geofencing argument and add it to the state map - CalculatedFieldState state = new GeofencingCalculatedFieldState(List.of("geofencingArgumentTest")); - state.updateState(mock(CalculatedFieldCtx.class), Map.of("geofencingArgumentTest", geofencingArgumentEntry)); + CalculatedFieldState state = new GeofencingCalculatedFieldState(DEVICE_ID); + state.update(mock(CalculatedFieldCtx.class), Map.of("geofencingArgumentTest", geofencingArgumentEntry)); // when CalculatedFieldStateProto proto = toProto(stateId, state); // then - CalculatedFieldState fromProto = CalculatedFieldUtils.fromProto(proto); + CalculatedFieldState fromProto = CalculatedFieldUtils.fromProto(stateId, proto); assertThat(fromProto) .usingRecursiveComparison() .ignoringFields("requiredArguments") diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index e26955d465..82ef7f4e8d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -74,7 +74,7 @@ public interface AlarmService extends EntityDaoService { AlarmApiCallResult acknowledgeAlarm(TenantId tenantId, AlarmId alarmId, long ackTs); - AlarmApiCallResult clearAlarm(TenantId tenantId, AlarmId alarmId, long clearTs, JsonNode details); + AlarmApiCallResult clearAlarm(TenantId tenantId, AlarmId alarmId, long clearTs, JsonNode details, boolean pushEvent); AlarmApiCallResult assignAlarm(TenantId tenantId, AlarmId alarmId, UserId assigneeId, long ts); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 48b07af29b..6d875f58bf 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -141,6 +141,8 @@ public enum MsgType { CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_ENTITY_ACTION_EVENT_MSG, + CF_ALARM_ACTION_MSG, CF_TELEMETRY_MSG, // Sent from queue to actor system; CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java index c05c0f121e..869ad659ac 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -16,12 +16,7 @@ package org.thingsboard.server.common.msg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.queue.TbCallback; public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { - default TbCallback getCallback() { - return TbCallback.EMPTY; - } - } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/aware/TenantAwareMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/aware/TenantAwareMsg.java index 4161940398..54ad749ceb 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/aware/TenantAwareMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/aware/TenantAwareMsg.java @@ -17,9 +17,14 @@ package org.thingsboard.server.common.msg.aware; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; public interface TenantAwareMsg extends TbActorMsg { TenantId getTenantId(); - + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index a05fdd5d36..5a8e348f5f 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -921,6 +921,7 @@ message CalculatedFieldStateProto { repeated SingleValueArgumentProto singleValueArguments = 3; repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; + AlarmStateProto alarmState = 6; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1721,10 +1722,13 @@ message ToEdgeEventNotificationMsg { message ToCalculatedFieldMsg { CalculatedFieldTelemetryMsgProto telemetryMsg = 1; CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; + EntityActionEventProto eventMsg = 3; + repeated string cfTypes = 4; } message ToCalculatedFieldNotificationMsg { CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 1; + repeated string cfTypes = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -1894,3 +1898,22 @@ message JobStatsMsg { message TaskResultProto { string value = 1; } + +message EntityActionEventProto { + EntityIdProto tenantId = 1; + EntityIdProto entityId = 2; + string entity = 3; + string action = 4; +} + +message AlarmStateProto { + repeated AlarmRuleStateProto createRuleStates = 1; + AlarmRuleStateProto clearRuleState = 2; +} + +message AlarmRuleStateProto { + string severity = 1; + int64 lastEventTs = 2; + int64 duration = 3; + int64 eventCount = 4; +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExpressionFunctionsUtil.java b/common/util/src/main/java/org/thingsboard/common/util/ExpressionUtils.java similarity index 87% rename from common/util/src/main/java/org/thingsboard/common/util/ExpressionFunctionsUtil.java rename to common/util/src/main/java/org/thingsboard/common/util/ExpressionUtils.java index b1753e7a17..96b45123a1 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ExpressionFunctionsUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ExpressionUtils.java @@ -15,13 +15,16 @@ */ package org.thingsboard.common.util; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import net.objecthunter.exp4j.function.Function; import net.objecthunter.exp4j.function.Functions; import java.util.ArrayList; import java.util.List; +import java.util.Set; -public class ExpressionFunctionsUtil { +public class ExpressionUtils { public static final List userDefinedFunctions = new ArrayList<>(); @@ -75,4 +78,13 @@ public class ExpressionFunctionsUtil { userDefinedFunctions.add(Functions.getBuiltinFunction("signum")); } + public static Expression createExpression(String expression, Set variables) { + return new ExpressionBuilder(expression) + .functions(userDefinedFunctions) + .implicitMultiplication(true) + .operator() + .variables(variables) + .build(); + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java b/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java index a924b0228e..0d9b8494f6 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java @@ -61,6 +61,37 @@ public class KvUtil { } } + public static Long getLongValue(KvEntry entry) { + switch (entry.getDataType()) { + case LONG -> { + return entry.getLongValue().orElse(null); + } + case DOUBLE -> { + return entry.getDoubleValue().map(Double::longValue).orElse(null); + } + case BOOLEAN -> { + return entry.getBooleanValue().map(b -> b ? 1L : 0L).orElse(null); + } + case STRING -> { + try { + return Long.parseLong(entry.getStrValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + } + case JSON -> { + try { + return Long.parseLong(entry.getJsonValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + } + default -> { + return null; + } + } + } + public static Boolean getBoolValue(KvEntry entry) { switch (entry.getDataType()) { case LONG: diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 5c3e4f3c79..3df4a64e73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -146,18 +146,26 @@ public class BaseAlarmService extends AbstractCachedEntityService (active && eval(alarmRule.getCondition(), data)) ? + AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; + case DURATION -> evalDuration(data, active); + case REPEATING -> evalRepeating(data, active); + }; } private boolean isActive(DataSnapshot data, long eventTs) { @@ -600,4 +596,5 @@ class AlarmRuleState { return null; } } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 90c35e59b1..c16d09969b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -59,7 +59,6 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntryAggWrapper; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; @@ -85,14 +84,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; @@ -246,7 +243,6 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { node.onMsg(ctx, msg2); verify(ctx).tellSuccess(msg2); verify(ctx).enqueueForTellNext(theMsg2, "Alarm Updated"); - } @Test From 5cf995d58126dd6736433ad096c42462341bb613 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 22 Sep 2025 16:48:02 +0300 Subject: [PATCH 003/122] Fix CF states tests --- .../service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java | 1 + .../service/cf/ctx/state/SimpleCalculatedFieldStateTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index bfc5bd36e5..d3bb7206f3 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -102,6 +102,7 @@ public class GeofencingCalculatedFieldStateTest { ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); state = new GeofencingCalculatedFieldState(ctx.getEntityId()); + state.init(ctx); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 30b79b0768..376b57d9d2 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -78,6 +78,7 @@ public class SimpleCalculatedFieldStateTest { ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getEntityId()); + state.init(ctx); } @Test From ab77b5d6b79dca8a3b5b9db7ee94adc325fb6b55 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 22 Sep 2025 17:55:07 +0300 Subject: [PATCH 004/122] Improvements for compatibility with PE --- .../cf/AbstractCalculatedFieldProcessingService.java | 5 ++--- .../server/service/cf/CalculatedFieldCache.java | 2 ++ .../server/service/cf/DefaultCalculatedFieldCache.java | 3 ++- .../cf/DefaultCalculatedFieldProcessingService.java | 7 ++++++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index b3632e7f26..6ecf7c5974 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -60,7 +60,7 @@ import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transfor @Data @Slf4j -public abstract class AbstractCalculatedFieldProcessingService implements CalculatedFieldProcessingService { +public abstract class AbstractCalculatedFieldProcessingService { protected final AttributesService attributesService; protected final TimeseriesService timeseriesService; @@ -84,8 +84,7 @@ public abstract class AbstractCalculatedFieldProcessingService implements Calcul protected abstract String getExecutorNamePrefix(); - @Override - public ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { + protected ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { Map> argFutures = switch (ctx.getCalculatedField().getType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false); case SIMPLE, SCRIPT, ALARM -> { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 5aac75a7c7..8dd3491942 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -45,4 +45,6 @@ public interface CalculatedFieldCache { void evict(CalculatedFieldId calculatedFieldId); + EntityId getProfileId(TenantId tenantId, EntityId entityId); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index f40aa503f7..0ef62c3568 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -211,7 +211,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); } - private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + @Override + public EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index dfc32741c8..f5c39ed288 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -60,7 +60,7 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @TbRuleEngineComponent @Service @Slf4j -public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedFieldProcessingService { +public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedFieldProcessingService implements CalculatedFieldProcessingService { private final TbClusterService clusterService; private final PartitionService partitionService; @@ -81,6 +81,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return "calculated-field-callback"; } + @Override + public ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchArguments(ctx, entityId); + } + @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only geofencing calculated fields supports dynamic arguments scheduled updates From 1a66f3973ea5b725a817b3a65ab8db0f4407c874 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Tue, 23 Sep 2025 14:10:03 +0300 Subject: [PATCH 005/122] Add repeating alarm condition support for Alarm rules CF --- ...CalculatedFieldEntityMessageProcessor.java | 24 +-- .../cf/AlarmCalculatedFieldResult.java | 17 +-- .../ctx/state/BaseCalculatedFieldState.java | 15 +- .../cf/ctx/state/CalculatedFieldCtx.java | 9 ++ .../cf/ctx/state/CalculatedFieldState.java | 4 +- .../ctx/state/ScriptCalculatedFieldState.java | 4 +- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../alarm/AlarmCalculatedFieldState.java | 64 ++++++-- .../cf/ctx/state/alarm/AlarmRuleState.java | 18 +++ .../GeofencingCalculatedFieldState.java | 19 ++- .../thingsboard/server/cf/AlarmRulesTest.java | 140 +++++++++++++++--- .../GeofencingCalculatedFieldStateTest.java | 25 ++-- .../state/ScriptCalculatedFieldStateTest.java | 7 +- .../state/SimpleCalculatedFieldStateTest.java | 11 +- .../utils/CalculatedFieldUtilsTest.java | 2 +- .../condition/DurationAlarmCondition.java | 2 + .../condition/RepeatingAlarmCondition.java | 2 + .../rule/engine/action/TbAlarmResult.java | 20 ++- 18 files changed, 293 insertions(+), 94 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 77665a1f10..1b03c9f7c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -122,7 +122,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); + log.debug("[{}] Processing entity init CF msg: {}", msg.getCtx().getCfId(), msg); var ctx = msg.getCtx(); CalculatedFieldState state; if (msg.getStateAction() == StateAction.RECREATE) { @@ -142,11 +142,12 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.init(ctx); } if (state.isSizeOk()) { - processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); + processStateIfReady(ctx, Collections.emptyMap(), Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); } else { throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); } } catch (Exception e) { + log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); if (e instanceof CalculatedFieldException cfe) { throw cfe; } @@ -176,7 +177,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing CF telemetry msg.", msg.getEntityId()); + log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); @@ -191,7 +192,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing CF link telemetry msg.", msg.getEntityId()); + log.trace("[{}] Processing CF link telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); var ctx = msg.getCtx(); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); @@ -213,6 +214,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } catch (Exception e) { + log.debug("[{}][{}] Failed to process linked CF telemetry msg: {}", entityId, ctx.getCfId(), msg, e); throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } } @@ -235,6 +237,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } catch (Exception e) { + log.debug("[{}][{}] Failed to process CF telemetry msg: {}", entityId, ctx.getCfId(), proto, e); if (e instanceof CalculatedFieldException cfe) { throw cfe; } @@ -305,10 +308,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } if (state.isSizeOk()) { - if (state.update(ctx, newArgValues) || justRestored) { + Map updatedArgs = state.update(newArgValues, ctx); + if (!updatedArgs.isEmpty() || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + processStateIfReady(ctx, updatedArgs, cfIdList, state, tbMsgId, tbMsgType, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -327,7 +331,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.init(ctx); Map arguments = fetchArguments(ctx); - state.update(ctx, arguments); + state.update(arguments, ctx); state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); states.put(ctx.getCfId(), state); @@ -343,12 +347,14 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return argumentsFuture.get(1, TimeUnit.MINUTES); } - private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + private void processStateIfReady(CalculatedFieldCtx ctx, Map updatedArgs, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + log.trace("[{}][{}] Processing state if ready. Current args: {}, updated args: {}", entityId, ctx.getCfId(), state.getArguments(), updatedArgs); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeChecked = false; try { if (ctx.isInitialized() && state.isReady()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); + log.trace("[{}][{}] Performing calculation. Updated args: {}", entityId, ctx.getCfId(), updatedArgs); + CalculatedFieldResult calculationResult = state.performCalculation(updatedArgs, ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeChecked = true; if (state.isSizeOk()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java index de48d05630..61f9cf37ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import lombok.Builder; import lombok.Data; +import lombok.RequiredArgsConstructor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; import org.thingsboard.server.common.data.DataConstants; @@ -25,16 +26,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import java.util.List; @Data @Builder +@RequiredArgsConstructor public class AlarmCalculatedFieldResult implements CalculatedFieldResult { private final TbAlarmResult alarmResult; - private final AlarmRuleState alarmRuleState; @Override public TbMsg toTbMsg(EntityId entityId, List cfIds) { @@ -49,14 +49,11 @@ public class AlarmCalculatedFieldResult implements CalculatedFieldResult { } else { metaData.putValue(DataConstants.IS_CLEARED_ALARM, Boolean.TRUE.toString()); } - switch (alarmRuleState.getCondition().getType()) { - case REPEATING -> { - metaData.putValue(DataConstants.ALARM_CONDITION_REPEATS, String.valueOf(alarmRuleState.getEventCount())); - } - case DURATION -> { - // TODO: schedule instead of duration - metaData.putValue(DataConstants.ALARM_CONDITION_DURATION, String.valueOf(alarmRuleState.getDuration())); - } + if (alarmResult.getConditionRepeats() != null) { + metaData.putValue(DataConstants.ALARM_CONDITION_REPEATS, String.valueOf(alarmResult.getConditionRepeats())); + } + if (alarmResult.getConditionDuration() != null) { + metaData.putValue(DataConstants.ALARM_CONDITION_DURATION, String.valueOf(alarmResult.getConditionDuration())); } return TbMsg.newMsg() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bd6d5b1a51..3baebc3dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,8 +45,8 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public boolean update(CalculatedFieldCtx ctx, Map argumentValues) { - boolean stateUpdated = false; + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = null; for (Map.Entry entry : argumentValues.entrySet()) { String key = entry.getKey(); @@ -65,13 +66,19 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } if (entryUpdated) { - stateUpdated = true; + if (updatedArguments == null) { + updatedArguments = new HashMap<>(argumentValues.size()); + } + updatedArguments.put(key, newEntry); updateLastUpdateTimestamp(newEntry); } } - return stateUpdated; + if (updatedArguments == null) { + updatedArguments = Collections.emptyMap(); + } + return updatedArguments; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 564e573765..1d008c77b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -455,4 +455,13 @@ public class CalculatedFieldCtx { return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; } + @Override + public String toString() { + return "CalculatedFieldCtx{" + + "cfId=" + cfId + + ", cfType=" + cfType + + ", entityId=" + entityId + + '}'; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index b397a8e87d..d7c061faba 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -56,11 +56,11 @@ public interface CalculatedFieldState { void init(CalculatedFieldCtx ctx); - boolean update(CalculatedFieldCtx ctx, Map arguments); + Map update(Map arguments, CalculatedFieldCtx ctx); void reset(CalculatedFieldCtx ctx); - ListenableFuture performCalculation(CalculatedFieldCtx ctx); + ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 13eaa69ca7..3b6f9b1f87 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import java.util.Map; + @Slf4j @EqualsAndHashCode(callSuper = true) public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @@ -41,7 +43,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { ListenableFuture resultFuture = ctx.evaluateTbelExpression(ctx.getExpression(), this); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 13aa3fe6fd..2dc8e1824a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import java.util.Map; + @EqualsAndHashCode(callSuper = true) public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @@ -48,7 +50,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { double expressionResult = ctx.evaluateSimpleExpression(ctx.getExpression(), this); Output output = ctx.getOutput(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 747846f655..f7da5f32fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; import org.thingsboard.server.common.data.audit.ActionType; @@ -104,23 +105,36 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { log.debug("Initialized create rule states {} and clear rule state {} for {}", createRuleStates, clearRuleState, ctx.getCalculatedField()); } + @Override + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + return super.update(argumentValues, ctx); + } + @Override public void reset(CalculatedFieldCtx ctx) { super.reset(ctx); } @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { + if (updatedArgs.isEmpty()) { + // FIXME: do we evaluate alarm rule (and increment event count) after arguments or expression change (state reinit)??? + return Futures.immediateFuture(new AlarmCalculatedFieldResult(null)); + } initCurrentAlarm(ctx); - AlarmCalculatedFieldResult result = createOrClearAlarms(state -> state.eval(ctx), ctx); - return Futures.immediateFuture(result); + TbAlarmResult result = createOrClearAlarms(state -> state.eval(ctx), ctx); + return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() + .alarmResult(result) + .build()); } // TODO: harvesting - public ListenableFuture performCalculation(long ts, CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(Map updatedArgs, long ts, CalculatedFieldCtx ctx) { initCurrentAlarm(ctx); - AlarmCalculatedFieldResult result = createOrClearAlarms(ruleState -> ruleState.eval(ts), ctx); - return Futures.immediateFuture(result); + TbAlarmResult result = createOrClearAlarms(ruleState -> ruleState.eval(ts), ctx); + return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() + .alarmResult(result) + .build()); } @SneakyThrows @@ -160,28 +174,33 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { createRuleStates.values().forEach(AlarmRuleState::clear); } - public AlarmCalculatedFieldResult createOrClearAlarms(Function evalFunction, CalculatedFieldCtx ctx) { + private TbAlarmResult createOrClearAlarms(Function evalFunction, + CalculatedFieldCtx ctx) { TbAlarmResult result = null; AlarmRuleState resultState = null; + AlarmRuleState.StateInfo resultStateInfo = null; + for (AlarmRuleState state : createRuleStates.values()) { AlarmEvalResult evalResult = evalFunction.apply(state); log.debug("Evaluated create rule {} with args {}. Result: {}", state, arguments, evalResult); - if (AlarmEvalResult.TRUE.equals(evalResult)) { + if (evalResult == AlarmEvalResult.TRUE) { resultState = state; break; - } else if (AlarmEvalResult.FALSE.equals(evalResult)) { + } else if (evalResult == AlarmEvalResult.FALSE) { clearAlarmState(state); } } if (resultState != null) { result = calculateAlarmResult(resultState, ctx); + resultStateInfo = resultState.getStateInfo(); log.debug("Alarm result for state {}: {}", resultState, result); clearAlarmState(clearRuleState); } else if (currentAlarm != null && clearRuleState != null) { AlarmEvalResult evalResult = evalFunction.apply(clearRuleState); log.debug("Evaluated clear rule {} with args {}. Result: {}", clearRuleState, arguments, evalResult); - if (AlarmEvalResult.TRUE.equals(evalResult)) { + if (evalResult == AlarmEvalResult.TRUE) { + resultStateInfo = clearRuleState.getStateInfo(); clearAlarmState(clearRuleState); for (AlarmRuleState state : createRuleStates.values()) { clearAlarmState(state); @@ -190,18 +209,23 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { ctx.getTenantId(), currentAlarm.getId(), System.currentTimeMillis(), createDetails(clearRuleState), true ); if (clearResult.isCleared()) { - result = new TbAlarmResult(false, false, true, clearResult.getAlarm()); + result = TbAlarmResult.builder() + .isCleared(true) + .alarm(clearResult.getAlarm()) + .build(); + addStateInfo(result, clearRuleState); resultState = clearRuleState; } currentAlarm = null; - } else if (AlarmEvalResult.FALSE.equals(evalResult)) { + } else if (evalResult == AlarmEvalResult.FALSE) { clearAlarmState(clearRuleState); } } - return AlarmCalculatedFieldResult.builder() - .alarmResult(result) - .alarmRuleState(resultState) - .build(); + if (result != null && resultState != null) { + result.setConditionRepeats(resultStateInfo.eventCount()); + result.setConditionDuration(resultStateInfo.duration()); + } + return result; } private void clearAlarmState(AlarmRuleState state) { @@ -265,6 +289,14 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } } + private void addStateInfo(TbAlarmResult alarmResult, AlarmRuleState ruleState) { + if (ruleState.getCondition().getType() == AlarmConditionType.REPEATING) { + alarmResult.setConditionRepeats(ruleState.getEventCount()); + } else if (ruleState.getCondition().getType() == AlarmConditionType.DURATION) { + alarmResult.setConditionDuration(ruleState.getDuration()); + } + } + private JsonNode createDetails(AlarmRuleState ruleState) { JsonNode alarmDetails; String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 04386c68f6..2e971ffebb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; import org.thingsboard.server.common.data.alarm.rule.condition.DurationAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.RepeatingAlarmCondition; @@ -194,6 +195,8 @@ public class AlarmRuleState { } private long getRequiredDurationInMs() { + // fixme timeUnit?? + return getValue(((DurationAlarmCondition) condition).getValue(), KvUtil::getLongValue); } @@ -219,6 +222,16 @@ public class AlarmRuleState { this.condition = alarmRule.getCondition(); } + public StateInfo getStateInfo() { + if (condition.getType() == AlarmConditionType.REPEATING) { + return new StateInfo(eventCount, null); + } else if (condition.getType() == AlarmConditionType.DURATION) { + return new StateInfo(null, duration); + } else { + return StateInfo.EMPTY; + } + } + @Override public String toString() { return "AlarmRuleState{" + @@ -230,4 +243,9 @@ public class AlarmRuleState { '}'; } + public record StateInfo(Long eventCount, Long duration) { + static final StateInfo EMPTY = new StateInfo(null, null); + + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java index b418dc73d8..ad31d23702 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java @@ -41,6 +41,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -68,8 +70,8 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public boolean update(CalculatedFieldCtx ctx, Map argumentValues) { - boolean stateUpdated = false; + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = null; for (var entry : argumentValues.entrySet()) { String key = entry.getKey(); @@ -103,14 +105,21 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { entryUpdated = existingEntry.updateEntry(newEntry); } if (entryUpdated) { - stateUpdated = true; + if (updatedArguments == null) { + updatedArguments = new HashMap<>(argumentValues.size()); + } + updatedArguments.put(key, newEntry); } } - return stateUpdated; + + if (updatedArguments == null) { + updatedArguments = Collections.emptyMap(); + } + return updatedArguments; } @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { double latitude = (double) arguments.get(ENTITY_ID_LATITUDE_ARGUMENT_KEY).getValue(); double longitude = (double) arguments.get(ENTITY_ID_LONGITUDE_ARGUMENT_KEY).getValue(); Coordinates entityCoordinates = new Coordinates(latitude, longitude); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 5cf674f6b6..166589038c 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.cf; +import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; @@ -23,11 +24,15 @@ import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; +import org.thingsboard.server.common.data.alarm.rule.condition.DurationAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.RepeatingAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.SimpleAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -56,6 +61,7 @@ import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; +@Slf4j @DaoSqlTest public class AlarmRulesTest extends AbstractControllerTest { @@ -79,15 +85,17 @@ public class AlarmRulesTest extends AbstractControllerTest { public void testCreateAndSeverityUpdateAndClear() throws Exception { Argument temperatureArgument = new Argument(); temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); Map arguments = Map.of( "temperature", temperatureArgument ); - Map createRules = Map.of( - AlarmSeverity.MAJOR, "return temperature >= 50;", - AlarmSeverity.CRITICAL, "return temperature >= 100;" + Map createRules = Map.of( + AlarmSeverity.MAJOR, new Condition("return temperature >= 50;", null, null), + AlarmSeverity.CRITICAL, new Condition("return temperature >= 100;", null, null) ); - String clearRule = "return temperature <= 25;"; + + Condition clearRule = new Condition("return temperature <= 25;", null, null); CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, clearRule); @@ -120,6 +128,71 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + /* + * todo: state restore (event count) + * */ + @Test + public void testCreateAlarmForRepeatingConditionOnTs() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + int eventsCountMajor = 5; + int eventsCountCritical = 10; + Map createRules = Map.of( + AlarmSeverity.MAJOR, new Condition("return temperature >= 50;", eventsCountMajor, null), + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", eventsCountCritical, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + for (int i = 0; i < 4; i++) { + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(10); + } + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + postTelemetry(deviceId, "{\"temperature\":50}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(5); + }); + } + + @Test + public void testCreateAlarmForRepeatingConditionOnAttribute() { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.ATTRIBUTE, AttributeScope.SHARED_SCOPE)); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + Map createRules = Map.of( + AlarmSeverity.MAJOR, "return temperature >= 50;", + AlarmSeverity.CRITICAL, "return temperature >= 100;" + ); + String clearRule = "return temperature <= 25;"; +// CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", +// arguments, createRules, clearRule); + } + + @Test + public void testCreateAlarmForDurationCondition() { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("powerConsumption", ArgumentType.TS_LATEST, null)); + Map arguments = Map.of( + "powerConsumption", temperatureArgument + ); + + +// CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", +// arguments, createRules, nu); + } + private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { TbAlarmResult alarmResult = getLatestAlarmResult(calculatedField.getId()); @@ -152,34 +225,61 @@ public class AlarmRulesTest extends AbstractControllerTest { private CalculatedField createAlarmCf(EntityId entityId, String alarmType, Map arguments, - Map createConditions, - String clearCondition) { + Map createConditions, + Condition clearCondition) { + Map createRules = new HashMap<>(); + createConditions.forEach((severity, condition) -> { + createRules.put(severity, toAlarmRule(condition)); + }); + AlarmRule clearRule = clearCondition != null ? toAlarmRule(clearCondition) : null; + CalculatedField calculatedField = createAlarmCf(entityId, alarmType, arguments, createRules, clearRule); + + CalculatedFieldDebugEvent debugEvent = await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getDebugEvents(calculatedField.getId(), 1), events -> !events.isEmpty()).get(0); + latestEventId = debugEvent.getId(); + return calculatedField; + } + + private CalculatedField createAlarmCf(EntityId entityId, + String alarmType, + Map arguments, + Map createRules, + AlarmRule clearRule) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(entityId); calculatedField.setName(alarmType); calculatedField.setType(CalculatedFieldType.ALARM); AlarmCalculatedFieldConfiguration configuration = new AlarmCalculatedFieldConfiguration(); configuration.setArguments(arguments); - configuration.setCreateRules(new HashMap<>()); - createConditions.forEach((severity, expression) -> { - configuration.getCreateRules().put(severity, toAlarmRule(expression)); - }); - configuration.setClearRule(toAlarmRule(clearCondition)); + configuration.setCreateRules(createRules); + configuration.setClearRule(clearRule); calculatedField.setConfiguration(configuration); calculatedField.setDebugSettings(DebugSettings.all()); return saveCalculatedField(calculatedField); } - private AlarmRule toAlarmRule(String conditionExpression) { - if (conditionExpression == null) { - return null; - } + private AlarmRule toAlarmRule(Condition condition) { AlarmRule rule = new AlarmRule(); - SimpleAlarmCondition condition = new SimpleAlarmCondition(); TbelAlarmConditionExpression expression = new TbelAlarmConditionExpression(); - expression.setExpression(conditionExpression); - condition.setExpression(expression); - rule.setCondition(condition); + expression.setExpression(condition.expression()); + if (condition.eventsCount() != null) { + RepeatingAlarmCondition alarmCondition = new RepeatingAlarmCondition(); + alarmCondition.setExpression(expression); + AlarmConditionValue count = new AlarmConditionValue<>(); + count.setStaticValue(condition.eventsCount()); + alarmCondition.setCount(count); + rule.setCondition(alarmCondition); + } else if (condition.durationMs() != null) { + DurationAlarmCondition alarmCondition = new DurationAlarmCondition(); + alarmCondition.setExpression(expression); + AlarmConditionValue duration = new AlarmConditionValue<>(); + duration.setStaticValue(condition.durationMs()); + alarmCondition.setValue(duration); + rule.setCondition(alarmCondition); + } else { + SimpleAlarmCondition alarmCondition = new SimpleAlarmCondition(); + alarmCondition.setExpression(expression); + rule.setCondition(alarmCondition); + } return rule; } @@ -188,4 +288,6 @@ public class AlarmRulesTest extends AbstractControllerTest { .map(e -> (CalculatedFieldDebugEvent) e).toList(); } + private record Condition(String expression, Integer eventsCount, Long durationMs) {} + } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index d3bb7206f3..b88442dc62 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -49,6 +49,7 @@ import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -118,7 +119,7 @@ public class GeofencingCalculatedFieldStateTest { )); Map newArgs = Map.of("allowedZones", geofencingAllowedZoneArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -132,21 +133,21 @@ public class GeofencingCalculatedFieldStateTest { @Test void testUpdateStateWithInvalidArgumentTypeForLatitudeArgument() { - assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for latitude argument: GEOFENCING. Only SINGLE_VALUE type is allowed."); } @Test void testUpdateStateWithInvalidArgumentTypeForLongitudeArgument() { - assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LONGITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(Map.of(ENTITY_ID_LONGITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for longitude argument: GEOFENCING. Only SINGLE_VALUE type is allowed."); } @Test void testUpdateStateWithInvalidArgumentTypeForGeofencingArgument() { - assertThatThrownBy(() -> state.update(ctx, Map.of("someArgumentName", latitudeArgEntry))) + assertThatThrownBy(() -> state.update(Map.of("someArgumentName", latitudeArgEntry), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for someArgumentName argument: SINGLE_VALUE. Only GEOFENCING type is allowed."); } @@ -157,7 +158,7 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 50.4760), 190L); Map newArgs = Map.of("latitude", newArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).isEqualTo(newArgs); @@ -169,7 +170,7 @@ public class GeofencingCalculatedFieldStateTest { Map newArgs = Map.of("allowedZones", geofencingAllowedZoneArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isFalse(); assertThat(state.getArguments()).isEqualTo(newArgs); @@ -179,7 +180,7 @@ public class GeofencingCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingSingleValueArgumentEntryWithValueOfAnotherType() { state.arguments = new HashMap<>(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, latitudeArgEntry)); - assertThatThrownBy(() -> state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry))) + assertThatThrownBy(() -> state.update(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, geofencingAllowedZoneArgEntry), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for single value argument entry: GEOFENCING"); } @@ -189,7 +190,7 @@ public class GeofencingCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingGeofencingValueArgumentEntryWithValueOfAnotherType() { state.arguments = new HashMap<>(Map.of("allowedZones", geofencingAllowedZoneArgEntry)); - assertThatThrownBy(() -> state.update(ctx, Map.of("allowedZones", latitudeArgEntry))) + assertThatThrownBy(() -> state.update(Map.of("allowedZones", latitudeArgEntry), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported argument entry type for geofencing argument entry: SINGLE_VALUE"); } @@ -255,7 +256,7 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude), ctx); TelemetryCalculatedFieldResult result2 = performCalculation(); @@ -327,7 +328,7 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude), ctx); TelemetryCalculatedFieldResult result2 = performCalculation(); @@ -399,7 +400,7 @@ public class GeofencingCalculatedFieldStateTest { SingleValueArgumentEntry newLongitude = new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", 30.5110), 166L); // move the device to new coordinates → leaves allowed, enters restricted - state.update(ctx, Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude)); + state.update(Map.of(ENTITY_ID_LATITUDE_ARGUMENT_KEY, newLatitude, ENTITY_ID_LONGITUDE_ARGUMENT_KEY, newLongitude), ctx); TelemetryCalculatedFieldResult result2 = performCalculation(); @@ -486,7 +487,7 @@ public class GeofencingCalculatedFieldStateTest { } private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { - return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + return (TelemetryCalculatedFieldResult) state.performCalculation(Collections.emptyMap(), ctx).get(); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 972d83f8a8..56fc2c1086 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.stats.DefaultStatsFactory; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -99,7 +100,7 @@ public class ScriptCalculatedFieldStateTest { state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -116,7 +117,7 @@ public class ScriptCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -228,7 +229,7 @@ public class ScriptCalculatedFieldStateTest { } private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { - return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + return (TelemetryCalculatedFieldResult) state.performCalculation(Collections.emptyMap(), ctx).get(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 376b57d9d2..00e3ed71f8 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -94,7 +95,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", key3ArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -112,7 +113,7 @@ public class SimpleCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); - boolean stateUpdated = state.update(ctx, newArgs); + boolean stateUpdated = !state.update(newArgs, ctx).isEmpty(); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); @@ -126,7 +127,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); - assertThatThrownBy(() -> state.update(ctx, newArgs)) + assertThatThrownBy(() -> state.update(newArgs, ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @@ -156,7 +157,7 @@ public class SimpleCalculatedFieldStateTest { "key3", key3ArgEntry )); - assertThatThrownBy(() -> state.performCalculation(ctx)) + assertThatThrownBy(() -> state.performCalculation(Collections.emptyMap(), ctx)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Argument 'key2' is not a number."); } @@ -271,7 +272,7 @@ public class SimpleCalculatedFieldStateTest { } private TelemetryCalculatedFieldResult performCalculation() throws InterruptedException, ExecutionException { - return (TelemetryCalculatedFieldResult) state.performCalculation(ctx).get(); + return (TelemetryCalculatedFieldResult) state.performCalculation(Collections.emptyMap(), ctx).get(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java index acdf7bbf36..40a7a14e1c 100644 --- a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java @@ -85,7 +85,7 @@ class CalculatedFieldUtilsTest { // Create cf state with the geofencing argument and add it to the state map CalculatedFieldState state = new GeofencingCalculatedFieldState(DEVICE_ID); - state.update(mock(CalculatedFieldCtx.class), Map.of("geofencingArgumentTest", geofencingArgumentEntry)); + state.update(Map.of("geofencingArgumentTest", geofencingArgumentEntry), mock(CalculatedFieldCtx.class)); // when CalculatedFieldStateProto proto = toProto(stateId, state); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java index 7656d63bc0..6210bd6b59 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java @@ -17,11 +17,13 @@ package org.thingsboard.server.common.data.alarm.rule.condition; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; import java.util.concurrent.TimeUnit; @Data @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) public class DurationAlarmCondition extends AlarmCondition { private TimeUnit unit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java index 9a57bb4631..cdf474c4dc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java @@ -17,9 +17,11 @@ package org.thingsboard.server.common.data.alarm.rule.condition; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; @Data @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) public class RepeatingAlarmCondition extends AlarmCondition { private AlarmConditionValue count; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAlarmResult.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAlarmResult.java index f594d69eab..d25846c984 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAlarmResult.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAlarmResult.java @@ -16,6 +16,7 @@ package org.thingsboard.rule.engine.action; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.alarm.Alarm; @@ -24,13 +25,18 @@ import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @Data @AllArgsConstructor @NoArgsConstructor +@Builder public class TbAlarmResult { + boolean isCreated; boolean isUpdated; boolean isSeverityUpdated; boolean isCleared; Alarm alarm; + Long conditionRepeats; + Long conditionDuration; + public TbAlarmResult(boolean isCreated, boolean isUpdated, boolean isCleared, Alarm alarm) { this.isCreated = isCreated; this.isUpdated = isUpdated; @@ -40,11 +46,13 @@ public class TbAlarmResult { public static TbAlarmResult fromAlarmResult(AlarmApiCallResult result) { boolean isSeverityChanged = result.isSeverityChanged(); - return new TbAlarmResult( - result.isCreated(), - result.isModified() && !isSeverityChanged, - isSeverityChanged, - result.isCleared(), - result.getAlarm()); + return TbAlarmResult.builder() + .isCreated(result.isCreated()) + .isUpdated(result.isModified() && !isSeverityChanged) + .isSeverityUpdated(isSeverityChanged) + .isCleared(result.isCleared()) + .alarm(result.getAlarm()) + .build(); } + } From 3e357e5e9bd13ada6633fe2b45fbc06b12ff658a Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 24 Sep 2025 11:24:36 +0300 Subject: [PATCH 006/122] Add initial duration alarm condition support for Alarm rules CF --- .../server/actors/ActorSystemContext.java | 6 +- .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 22 ++++- ...alculatedFieldManagerMessageProcessor.java | 23 +++++ .../CalculatedFieldReevaluateMsg.java | 35 ++++++++ .../cf/ctx/state/CalculatedFieldCtx.java | 2 + .../alarm/AlarmCalculatedFieldState.java | 22 ++--- .../cf/ctx/state/alarm/AlarmRuleState.java | 88 ++++++++++--------- .../src/main/resources/thingsboard.yml | 3 + .../thingsboard/server/cf/AlarmRulesTest.java | 65 ++++++++------ .../alarm/rule/condition/AlarmCondition.java | 1 + .../condition/DurationAlarmCondition.java | 5 ++ .../condition/RepeatingAlarmCondition.java | 4 + .../common/data/cf/CalculatedField.java | 4 + .../AlarmCalculatedFieldConfiguration.java | 14 +++ .../CalculatedFieldConfiguration.java | 4 + .../server/common/msg/MsgType.java | 3 +- 17 files changed, 219 insertions(+), 85 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index ea46ce86eb..8a6c17726c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -654,6 +654,10 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; + @Value("${actors.alarms.reevaluation_interval:60}") + @Getter + private long alarmsReevaluationInterval; + @Autowired @Getter private MqttClientSettings mqttClientSettings; @@ -857,7 +861,7 @@ public class ActorSystemContext { private boolean checkLimits(TenantId tenantId) { if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled() && - !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) { + !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) { log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index bf24c8ff84..e0f70509a4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -78,6 +78,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DYNAMIC_ARGUMENTS_REFRESH_MSG: processor.process((EntityCalculatedFieldDynamicArgumentsRefreshMsg) msg); break; + case CF_REEVALUATE_MSG: + processor.process((CalculatedFieldReevaluateMsg) msg); + break; case CF_ALARM_ACTION_MSG: processor.process((CalculatedFieldAlarmActionMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1b03c9f7c5..ee425bbf90 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -142,7 +142,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.init(ctx); } if (state.isSizeOk()) { - processStateIfReady(ctx, Collections.emptyMap(), Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); } else { throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); } @@ -257,6 +257,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); } + public void process(CalculatedFieldReevaluateMsg msg) throws CalculatedFieldException { + CalculatedFieldId cfId = msg.getCfCtx().getCfId(); + CalculatedFieldState state = states.get(cfId); + if (state == null) { + log.debug("[{}][{}] Failed to find CF state for entity to handle {}", entityId, cfId, msg); + } else { + if (state.isSizeOk()) { + log.debug("[{}][{}] Reevaluating CF state", entityId, cfId); + processStateIfReady(state, null, msg.getCfCtx(), Collections.singletonList(cfId), null, null, msg.getCallback()); + } else { + throw new RuntimeException(msg.getCfCtx().getSizeExceedsLimitMessage()); + } + } + } + public void process(CalculatedFieldAlarmActionMsg msg) { log.debug("[{}] Processing alarm action event msg: {}", entityId, msg); states.values().forEach(state -> { @@ -312,7 +327,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (!updatedArgs.isEmpty() || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, updatedArgs, cfIdList, state, tbMsgId, tbMsgType, callback); + processStateIfReady(state, updatedArgs, ctx, cfIdList, tbMsgId, tbMsgType, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -347,7 +362,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return argumentsFuture.get(1, TimeUnit.MINUTES); } - private void processStateIfReady(CalculatedFieldCtx ctx, Map updatedArgs, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + private void processStateIfReady(CalculatedFieldState state, Map updatedArgs, CalculatedFieldCtx ctx, + List cfIdList, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { log.trace("[{}][{}] Processing state if ready. Current args: {}, updated args: {}", entityId, ctx.getCfId(), state.getArguments(), updatedArgs); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeChecked = false; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 43a21b196a..299a2bc8b9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -78,6 +78,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> cfDynamicArgumentsRefreshTasks = new ConcurrentHashMap<>(); + private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; @@ -118,6 +119,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityIdCalculatedFieldLinks.clear(); cfDynamicArgumentsRefreshTasks.values().forEach(future -> future.cancel(true)); cfDynamicArgumentsRefreshTasks.clear(); + if (cfsReevaluationTask != null) { + cfsReevaluationTask.cancel(true); + cfsReevaluationTask = null; + } ctx.stop(ctx.getSelf()); } @@ -125,6 +130,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); initEntityProfileCache(); initCalculatedFields(); + scheduleCfsReevaluation(); msg.getCallback().onSuccess(); } @@ -143,6 +149,23 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + private void scheduleCfsReevaluation() { + cfsReevaluationTask = systemContext.getScheduler().scheduleWithFixedDelay(() -> { + try { + calculatedFields.values().forEach(cf -> { + if (cf.isRequiresScheduledReevaluation()) { + applyToTargetCfEntityActors(cf, TbCallback.EMPTY, (entityId, callback) -> { + log.debug("[{}][{}] Pushing scheduled CF reevaluate msg", entityId, cf.getCfId()); + getOrCreateActor(entityId).tell(new CalculatedFieldReevaluateMsg(tenantId, cf)); + }); + } + }); + } catch (Exception e) { + log.warn("[{}] Failed to trigger CFs reevaluation", tenantId, e); + } + }, systemContext.getAlarmsReevaluationInterval(), systemContext.getAlarmsReevaluationInterval(), TimeUnit.SECONDS); + } + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java new file mode 100644 index 0000000000..a0b75d1a72 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldReevaluateMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx cfCtx; + + @Override + public MsgType getMsgType() { + return MsgType.CF_REEVALUATE_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1d008c77b8..13f5c8e6c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -77,6 +77,7 @@ public class CalculatedFieldCtx { private Output output; private String expression; private boolean useLatestTs; + private boolean requiresScheduledReevaluation; private TbelInvokeService tbelInvokeService; private RelationService relationService; @@ -140,6 +141,7 @@ public class CalculatedFieldCtx { }); } } + this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); this.alarmService = systemContext.getAlarmService(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index f7da5f32fa..bc40af2568 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -117,21 +117,15 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { - if (updatedArgs.isEmpty()) { - // FIXME: do we evaluate alarm rule (and increment event count) after arguments or expression change (state reinit)??? - return Futures.immediateFuture(new AlarmCalculatedFieldResult(null)); - } initCurrentAlarm(ctx); - TbAlarmResult result = createOrClearAlarms(state -> state.eval(ctx), ctx); - return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() - .alarmResult(result) - .build()); - } - - // TODO: harvesting - public ListenableFuture performCalculation(Map updatedArgs, long ts, CalculatedFieldCtx ctx) { - initCurrentAlarm(ctx); - TbAlarmResult result = createOrClearAlarms(ruleState -> ruleState.eval(ts), ctx); + TbAlarmResult result = createOrClearAlarms(state -> { + if (updatedArgs != null) { + boolean newEvent = !updatedArgs.isEmpty(); + return state.eval(newEvent, ctx); + } else { + return state.eval(System.currentTimeMillis()); + } + }, ctx); return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() .alarmResult(result) .build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 2e971ffebb..fc209110fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -64,21 +64,21 @@ public class AlarmRuleState { this.state = state; } - public AlarmEvalResult eval(CalculatedFieldCtx ctx) { + public AlarmEvalResult eval(boolean newEvent, CalculatedFieldCtx ctx) { // on event or config change boolean active = isActive(state.getLatestTimestamp()); return switch (condition.getType()) { - case SIMPLE -> (active && eval(condition.getExpression(), ctx)) ? AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; + case SIMPLE -> evalSimple(active, ctx); case DURATION -> evalDuration(active, ctx); - case REPEATING -> evalRepeating(active, ctx); + case REPEATING -> evalRepeating(active, newEvent, ctx); }; } - public AlarmEvalResult eval(long ts) { + public AlarmEvalResult eval(long ts) { // on schedule switch (condition.getType()) { - case SIMPLE: - case REPEATING: + case SIMPLE, REPEATING -> { return AlarmEvalResult.NOT_YET_TRUE; - case DURATION: + } + case DURATION -> { long requiredDurationInMs = getRequiredDurationInMs(); if (requiredDurationInMs > 0 && lastEventTs > 0 && ts > lastEventTs) { long duration = this.duration + (ts - lastEventTs); @@ -88,8 +88,43 @@ public class AlarmRuleState { return AlarmEvalResult.FALSE; } } - default: - return AlarmEvalResult.FALSE; + } + } + return AlarmEvalResult.FALSE; + } + + private AlarmEvalResult evalSimple(boolean active, CalculatedFieldCtx ctx) { + return (active && eval(condition.getExpression(), ctx)) ? + AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; + } + + private AlarmEvalResult evalRepeating(boolean active, boolean newEvent, CalculatedFieldCtx ctx) { + if (active && eval(condition.getExpression(), ctx)) { + if (newEvent) { + eventCount++; + } + long requiredRepeats = getIntValue(((RepeatingAlarmCondition) condition).getCount()); + return eventCount >= requiredRepeats ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; + } + } + + private AlarmEvalResult evalDuration(boolean active, CalculatedFieldCtx ctx) { + if (active && eval(condition.getExpression(), ctx)) { + if (lastEventTs > 0) { + if (state.getLatestTimestamp() > lastEventTs) { + duration = duration + (state.getLatestTimestamp() - lastEventTs); + lastEventTs = state.getLatestTimestamp(); + } + } else { + lastEventTs = state.getLatestTimestamp(); + duration = 0L; + } + long requiredDurationInMs = getRequiredDurationInMs(); + return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; } } @@ -162,42 +197,13 @@ public class AlarmRuleState { duration = 0L; } - private AlarmEvalResult evalRepeating(boolean active, CalculatedFieldCtx ctx) { - if (active && eval(condition.getExpression(), ctx)) { - eventCount++; - long requiredRepeats = getIntValue(((RepeatingAlarmCondition) condition).getCount()); - return eventCount >= requiredRepeats ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; - } else { - return AlarmEvalResult.FALSE; - } - } - - private AlarmEvalResult evalDuration(boolean active, CalculatedFieldCtx ctx) { - if (active && eval(condition.getExpression(), ctx)) { - if (lastEventTs > 0) { - if (state.getLatestTimestamp() > lastEventTs) { - duration = duration + (state.getLatestTimestamp() - lastEventTs); - lastEventTs = state.getLatestTimestamp(); - } - } else { - lastEventTs = state.getLatestTimestamp(); - duration = 0L; - } - long requiredDurationInMs = getRequiredDurationInMs(); - return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; - } else { - return AlarmEvalResult.FALSE; - } - } - private Integer getIntValue(AlarmConditionValue value) { return getValue(value, entry -> Optional.ofNullable(KvUtil.getLongValue(entry)).map(Long::intValue).orElse(null)); } private long getRequiredDurationInMs() { - // fixme timeUnit?? - - return getValue(((DurationAlarmCondition) condition).getValue(), KvUtil::getLongValue); + DurationAlarmCondition durationCondition = (DurationAlarmCondition) condition; + return durationCondition.getUnit().toMillis(getValue(durationCondition.getValue(), KvUtil::getLongValue)); } private boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { @@ -226,7 +232,7 @@ public class AlarmRuleState { if (condition.getType() == AlarmConditionType.REPEATING) { return new StateInfo(eventCount, null); } else if (condition.getType() == AlarmConditionType.DURATION) { - return new StateInfo(null, duration); + return new StateInfo(null, duration + (System.currentTimeMillis() - lastEventTs)); } else { return StateInfo.EMPTY; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 68cd479411..f2d15f53a0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -526,6 +526,9 @@ actors: configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" + alarms: + # Interval in seconds to re-evaluate Alarm rules with duration condition + reevaluation_interval: "${ACTORS_ALARMS_REEVALUATION_INTERVAL_SEC:60}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 166589038c..1bfb2bf875 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -20,11 +20,11 @@ import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -63,6 +63,9 @@ import static org.testcontainers.shaded.org.awaitility.Awaitility.await; @Slf4j @DaoSqlTest +@TestPropertySource(properties = { + "actors.alarms.reevaluation_interval=1" +}) public class AlarmRulesTest extends AbstractControllerTest { @MockitoSpyBean @@ -129,10 +132,10 @@ public class AlarmRulesTest extends AbstractControllerTest { } /* - * todo: state restore (event count) - * */ + * todo: state restore (event count) + * */ @Test - public void testCreateAlarmForRepeatingConditionOnTs() throws Exception { + public void testCreateAlarmForRepeatingCondition() throws Exception { Argument temperatureArgument = new Argument(); temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); temperatureArgument.setDefaultValue("0"); @@ -161,36 +164,47 @@ public class AlarmRulesTest extends AbstractControllerTest { assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); assertThat(alarmResult.getConditionRepeats()).isEqualTo(5); }); + + for (int i = 0; i < 5; i++) { + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(10); + } + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isSeverityUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(10); + }); } @Test - public void testCreateAlarmForRepeatingConditionOnAttribute() { - Argument temperatureArgument = new Argument(); - temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.ATTRIBUTE, AttributeScope.SHARED_SCOPE)); + public void testCreateAlarmForDurationCondition() throws Exception { + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("powerConsumption", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("0"); Map arguments = Map.of( - "temperature", temperatureArgument + "powerConsumption", argument ); - Map createRules = Map.of( - AlarmSeverity.MAJOR, "return temperature >= 50;", - AlarmSeverity.CRITICAL, "return temperature >= 100;" - ); - String clearRule = "return temperature <= 25;"; -// CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", -// arguments, createRules, clearRule); - } - - @Test - public void testCreateAlarmForDurationCondition() { - Argument temperatureArgument = new Argument(); - temperatureArgument.setRefEntityKey(new ReferencedEntityKey("powerConsumption", ArgumentType.TS_LATEST, null)); - Map arguments = Map.of( - "powerConsumption", temperatureArgument + long createDurationMs = 5000L; + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return powerConsumption >= 3000;", null, createDurationMs) ); + long clearDurationMs = 2000L; + Condition clearRule = new Condition("return powerConsumption < 3000;", null, createDurationMs); + CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 3 seconds", + arguments, createRules, clearRule); + postTelemetry(deviceId, "{\"powerConsumption\":3500}"); + Thread.sleep(createDurationMs - 2000); + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); -// CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", -// arguments, createRules, nu); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionDuration()).isBetween(createDurationMs, createDurationMs + 2000); + }); } private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { @@ -271,6 +285,7 @@ public class AlarmRulesTest extends AbstractControllerTest { } else if (condition.durationMs() != null) { DurationAlarmCondition alarmCondition = new DurationAlarmCondition(); alarmCondition.setExpression(expression); + alarmCondition.setUnit(TimeUnit.MILLISECONDS); AlarmConditionValue duration = new AlarmConditionValue<>(); duration.setStaticValue(condition.durationMs()); alarmCondition.setValue(duration); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java index 36b03b62ae..a13de08480 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java @@ -41,6 +41,7 @@ public abstract class AlarmCondition { @NotNull @Valid private AlarmConditionExpression expression; + @Valid private AlarmConditionValue schedule; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java index 6210bd6b59..22733ab78d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/DurationAlarmCondition.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -26,7 +28,10 @@ import java.util.concurrent.TimeUnit; @ToString(callSuper = true) public class DurationAlarmCondition extends AlarmCondition { + @NotNull private TimeUnit unit; + @Valid + @NotNull private AlarmConditionValue value; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java index cdf474c4dc..7919a6a22a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/RepeatingAlarmCondition.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -24,6 +26,8 @@ import lombok.ToString; @ToString(callSuper = true) public class RepeatingAlarmCondition extends AlarmCondition { + @Valid + @NotNull private AlarmConditionValue count; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 3b2ddf0627..9dd92294db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -18,6 +18,8 @@ package org.thingsboard.server.common.data.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSetter; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -64,6 +66,8 @@ public class CalculatedField extends BaseData implements HasN @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + @Valid + @NotNull private CalculatedFieldConfiguration configuration; @Getter @Setter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java index bb0834b3a7..c2925d5ed6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.List; @@ -26,9 +29,14 @@ import java.util.Map; @Data public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + @Valid + @NotEmpty private Map arguments; + @Valid + @NotEmpty private Map createRules; + @Valid private AlarmRule clearRule; private boolean propagate; @@ -51,4 +59,10 @@ public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculat } + @Override + public boolean requiresScheduledReevaluation() { + return createRules.values().stream().anyMatch(rule -> rule.getCondition().getType() == AlarmConditionType.DURATION) || + (clearRule != null && clearRule.getCondition().getType() == AlarmConditionType.DURATION); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 7b608192db..d3622a2dcf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -72,4 +72,8 @@ public interface CalculatedFieldConfiguration { .collect(Collectors.toList()); } + default boolean requiresScheduledReevaluation() { + return false; + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 6d875f58bf..fca3632ee8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -152,7 +152,8 @@ public enum MsgType { CF_ENTITY_DELETE_MSG, CF_DYNAMIC_ARGUMENTS_REFRESH_MSG, - CF_ENTITY_DYNAMIC_ARGUMENTS_REFRESH_MSG; + CF_ENTITY_DYNAMIC_ARGUMENTS_REFRESH_MSG, + CF_REEVALUATE_MSG; @Getter private final boolean ignoreOnStart; From bf3e6dce7647c3469bde834d737ecee86f5b5fe6 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Tue, 30 Sep 2025 11:04:07 +0300 Subject: [PATCH 007/122] CF: Current customer dynamic source support --- .../server/actors/ActorSystemContext.java | 5 + .../CalculatedFieldArgumentResetMsg.java | 37 +++++++ .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 34 ++++--- ...alculatedFieldManagerMessageProcessor.java | 96 ++++++++++++++++++- ...tractCalculatedFieldProcessingService.java | 61 +++++++++--- .../service/cf/CalculatedFieldCache.java | 11 +++ .../cf/DefaultCalculatedFieldCache.java | 43 +++++++++ ...faultCalculatedFieldProcessingService.java | 13 +-- .../DefaultCalculatedFieldQueueService.java | 22 ++++- .../server/service/cf/OwnerService.java | 67 +++++++++++++ .../cf/ctx/state/CalculatedFieldCtx.java | 76 +++++++++++++-- .../queue/DefaultTbClusterService.java | 2 + .../queue/DefaultTbEdgeConsumerService.java | 3 +- .../processing/AbstractConsumerService.java | 16 ++++ .../utils/CalculatedFieldArgumentUtils.java | 7 ++ .../thingsboard/server/cf/AlarmRulesTest.java | 39 +++++++- .../cf/CalculatedFieldIntegrationTest.java | 6 -- .../server/controller/AbstractWebTest.java | 10 +- .../server/common/data/Device.java | 6 ++ .../common/data/ProfileEntityIdInfo.java | 12 ++- .../server/common/data/asset/Asset.java | 7 ++ .../data/cf/configuration/Argument.java | 8 ++ .../BaseCalculatedFieldConfiguration.java | 4 +- ...eofencingCalculatedFieldConfiguration.java | 2 +- .../geofencing/ZoneGroupConfiguration.java | 11 ++- .../data/cf/configuration/ArgumentTest.java | 13 ++- ...ncingCalculatedFieldConfigurationTest.java | 6 +- .../ZoneGroupConfigurationTest.java | 15 ++- .../server/common/msg/MsgType.java | 1 + .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../CalculatedFieldDataValidator.java | 2 +- .../device/DefaultNativeAssetRepository.java | 32 ++++--- .../device/DefaultNativeDeviceRepository.java | 31 +++--- 36 files changed, 603 insertions(+), 107 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldArgumentResetMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 8a6c17726c..d000b46c89 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -116,6 +116,7 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.OwnerService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -566,6 +567,10 @@ public class ActorSystemContext { @Getter private JobManager jobManager; + @Autowired + @Getter + private OwnerService ownerService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private int maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldArgumentResetMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldArgumentResetMsg.java new file mode 100644 index 0000000000..8b5927827e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldArgumentResetMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldArgumentResetMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ARGUMENT_RESET_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index e0f70509a4..0db3cfdb4c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -84,6 +84,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ALARM_ACTION_MSG: processor.process((CalculatedFieldAlarmActionMsg) msg); break; + case CF_ARGUMENT_RESET_MSG: + processor.process((CalculatedFieldArgumentResetMsg) msg); + break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ee425bbf90..ecfb258cd1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -155,6 +155,24 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } + public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { + log.debug("[{}] Processing CF argument reset msg.", entityId); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + try { + Map dynamicSourceArgs = ctx.getArguments().entrySet().stream() + .filter(entry -> entry.getValue().hasOwnerSource()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, dynamicSourceArgs); + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + + processArgumentValuesUpdate(ctx, Collections.singletonList(ctx.getCfId()), callback, fetchedArgs, null, null); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + public void process(CalculatedFieldEntityDeleteMsg msg) { log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { @@ -422,11 +440,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - var argNames = ctx.getLinkedEntityArguments().get(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } - return mapToArguments(argNames, data); + return mapToArguments(ctx.getLinkedAndDynamicArgs(entityId), data); } private Map mapToArguments(Map argNames, List data) { @@ -454,11 +468,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { - var argNames = ctx.getLinkedEntityArguments().get(entityId); + var argNames = ctx.getLinkedAndDynamicArgs(entityId); if (argNames.isEmpty()) { return Collections.emptyMap(); } - List geofencingArgumentNames = ctx.getLinkedEntityGeofencingArgumentNames(); + List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); } @@ -480,11 +494,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { - var argNames = ctx.getLinkedEntityArguments().get(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } - return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(ctx.getLinkedAndDynamicArgs(entityId), ctx.getArguments(), scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 299a2bc8b9..6aa501cb10 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -49,6 +49,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.OwnerService; import org.thingsboard.server.service.cf.cache.TenantEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -58,8 +59,10 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -77,6 +80,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); + private final Map> ownerEntities = new HashMap<>(); private final Map> cfDynamicArgumentsRefreshTasks = new ConcurrentHashMap<>(); private ScheduledFuture cfsReevaluationTask; @@ -88,6 +92,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; + private final OwnerService ownerService; private final TbQueueCalculatedFieldSettings cfSettings; protected final TenantId tenantId; @@ -103,6 +108,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); + this.ownerService = systemContext.getOwnerService(); this.cfSettings = systemContext.getCalculatedFieldSettings(); this.tenantId = tenantId; } @@ -128,7 +134,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onCacheInitMsg(CalculatedFieldCacheInitMsg msg) { log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); - initEntityProfileCache(); + initEntitiesCache(); initCalculatedFields(); scheduleCfsReevaluation(); msg.getCallback().onSuccess(); @@ -251,6 +257,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (profileId != null) { entityProfileCache.add(profileId, entityId); } + updateEntityOwner(entityId); if (!isMyPartition(entityId, callback)) { return; } @@ -283,11 +290,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + } else if (msg.isOwnerChanged()) { + onEntityOwnerChanged(msg, callback); + } else { + callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { entityProfileCache.removeEntityId(msg.getEntityId()); + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); @@ -415,8 +427,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); log.debug("Received telemetry msg from entity [{}]", entityId); - // 2 = 1 for CF processing + 1 for links processing - MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); + // 3 = 1 for CF processing + 1 for links processing + 1 for owner entity processing + MultipleTbCallback callback = new MultipleTbCallback(3, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); @@ -434,6 +446,17 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + // process all cfs related to owner entity + if (entityId.getEntityType().isOneOf(EntityType.TENANT, EntityType.CUSTOMER)) { + List ownerCFs = filterOwnerEntitiesCFs(msg); + if (!ownerCFs.isEmpty()) { + cfExecService.pushMsgToLinks(msg, ownerCFs, callback); + } else { + callback.onSuccess(); + } + } else { + callback.onSuccess(); + } } public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { @@ -456,6 +479,31 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + private void onEntityOwnerChanged(ComponentLifecycleMsg msg, TbCallback msgCallback) { + EntityId entityId = msg.getEntityId(); + log.debug("Received changed owner msg from entity [{}]", entityId); + updateEntityOwner(entityId); + List cfs = new ArrayList<>(); + cfs.addAll(getCalculatedFieldsByEntityId(entityId)); + cfs.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + if (cfs.isEmpty()) { + msgCallback.onSuccess(); + return; + } + MultipleTbCallback callback = new MultipleTbCallback(cfs.size(), msgCallback); + cfs.forEach(cf -> { + if (isMyPartition(entityId, callback)) { + if (cf.hasCurrentOwnerSourceArguments()) { + CalculatedFieldArgumentResetMsg argResetMsg = new CalculatedFieldArgumentResetMsg(tenantId, cf, callback); + log.debug("Pushing CF argument reset msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(argResetMsg); + } else { + callback.onSuccess(); + } + } + }); + } + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); var proto = msg.getProto(); @@ -469,6 +517,27 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private List filterOwnerEntitiesCFs(CalculatedFieldTelemetryMsg msg) { + Set entities = getOwnerEntities(msg.getEntityId()); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var entityId : entities) { + var ownerEntityCFs = getCalculatedFieldsByEntityId(entityId); + for (var ctx : ownerEntityCFs) { + if (ctx.dynamicSourceMatches(proto)) { + result.add(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId)); + } + } + var ownerEntityProfileCFs = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + for (var ctx : ownerEntityProfileCFs) { + if (ctx.dynamicSourceMatches(proto)) { + result.add(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId)); + } + } + } + return result; + } + private List getCalculatedFieldsByEntityId(EntityId entityId) { if (entityId == null) { return Collections.emptyList(); @@ -491,6 +560,17 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private Set getOwnerEntities(EntityId entityId) { + if (entityId == null) { + return Collections.emptySet(); + } + var result = ownerEntities.get(entityId); + if (result == null) { + result = Collections.emptySet(); + } + return result; + } + private void scheduleDynamicArgumentsRefreshTaskForCfIfNeeded(CalculatedFieldCtx cfCtx) { CalculatedField cf = cfCtx.getCalculatedField(); if (!(cf.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledCfConfig)) { @@ -623,12 +703,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); } - private void initEntityProfileCache() { + private void initEntitiesCache() { PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); + ownerEntities.computeIfAbsent(idInfo.getOwnerId(), ownerId -> new HashSet<>()).add(idInfo.getEntityId()); } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } @@ -638,12 +719,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.trace("Processing asset record: {}", idInfo); try { entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); + ownerEntities.computeIfAbsent(idInfo.getOwnerId(), ownerId -> new HashSet<>()).add(idInfo.getEntityId()); } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } } } + private void updateEntityOwner(EntityId entityId) { + ownerEntities.values().forEach(entities -> entities.remove(entityId)); + EntityId owner = ownerService.getOwner(tenantId, entityId); + ownerEntities.computeIfAbsent(owner, ownerId -> new HashSet<>()).add(entityId); + } + private void applyToTargetCfEntityActors(CalculatedFieldCtx ctx, TbCallback callback, BiConsumer action) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 6ecf7c5974..343db5286f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -19,11 +19,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationQueryDynamicSourceConfiguration; @@ -45,6 +47,7 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,6 +58,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @@ -66,6 +70,7 @@ public abstract class AbstractCalculatedFieldProcessingService { protected final TimeseriesService timeseriesService; protected final ApiLimitService apiLimitService; protected final RelationService relationService; + protected final OwnerService ownerService; protected ListeningExecutorService calculatedFieldCallbackExecutor; @@ -84,14 +89,14 @@ public abstract class AbstractCalculatedFieldProcessingService { protected abstract String getExecutorNamePrefix(); - protected ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { + protected ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { Map> argFutures = switch (ctx.getCalculatedField().getType()) { - case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false); + case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM -> { Map> futures = new HashMap<>(); for (var entry : ctx.getArguments().entrySet()) { - var argEntityId = resolveEntityId(entityId, entry.getValue()); - var argValueFuture = fetchArgumentValue(ctx.getTenantId(), argEntityId, entry.getValue(), System.currentTimeMillis()); + var argEntityId = resolveEntityId(ctx.getTenantId(), entityId, entry.getValue()); + var argValueFuture = fetchArgumentValue(ctx.getTenantId(), argEntityId, entry.getValue(), ts); futures.put(entry.getKey(), argValueFuture); } yield futures; @@ -102,8 +107,14 @@ public abstract class AbstractCalculatedFieldProcessingService { MoreExecutors.directExecutor()); } - protected EntityId resolveEntityId(EntityId entityId, Argument argument) { - return argument.getRefEntityId() != null ? argument.getRefEntityId() : entityId; + protected EntityId resolveEntityId(TenantId tenantId, EntityId entityId, Argument argument) { + if (argument.getRefEntityId() != null) { + return argument.getRefEntityId(); + } + if (!argument.hasOwnerSource()) { + return entityId; + } + return resolveOwnerArgument(tenantId, entityId, argument); } protected Map resolveArgumentFutures(Map> argFutures) { @@ -123,18 +134,18 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } - protected Map> fetchGeofencingCalculatedFieldArguments(CalculatedFieldCtx ctx, EntityId entityId, boolean dynamicArgumentsOnly) { + protected Map> fetchGeofencingCalculatedFieldArguments(CalculatedFieldCtx ctx, EntityId entityId, boolean dynamicArgumentsOnly, long startTs) { Map> argFutures = new HashMap<>(); Set> entries = ctx.getArguments().entrySet(); if (dynamicArgumentsOnly) { entries = entries.stream() - .filter(entry -> entry.getValue().hasDynamicSource()) + .filter(entry -> entry.getValue().hasRelationQuerySource()) .collect(Collectors.toSet()); } for (var entry : entries) { switch (entry.getKey()) { case ENTITY_ID_LATITUDE_ARGUMENT_KEY, ENTITY_ID_LONGITUDE_ARGUMENT_KEY -> - argFutures.put(entry.getKey(), fetchArgumentValue(ctx.getTenantId(), entityId, entry.getValue(), System.currentTimeMillis())); + argFutures.put(entry.getKey(), fetchArgumentValue(ctx.getTenantId(), entityId, entry.getValue(), startTs)); default -> { var resolvedEntityIdsFuture = resolveGeofencingEntityIds(ctx.getTenantId(), entityId, entry); argFutures.put(entry.getKey(), Futures.transformAsync(resolvedEntityIdsFuture, resolvedEntityIds -> @@ -155,6 +166,14 @@ public abstract class AbstractCalculatedFieldProcessingService { } var refDynamicSourceConfiguration = value.getRefDynamicSourceConfiguration(); return switch (refDynamicSourceConfiguration.getType()) { + case CURRENT_CUSTOMER -> { + EntityId resolved = resolveOwnerArgument(tenantId, entityId, value); + if (resolved != null) { + yield Futures.immediateFuture(List.of(resolved)); + } else { + yield Futures.immediateFuture(Collections.emptyList()); + } + } case RELATION_QUERY -> { var configuration = (RelationQueryDynamicSourceConfiguration) refDynamicSourceConfiguration; if (configuration.isSimpleRelation()) { @@ -170,7 +189,23 @@ public abstract class AbstractCalculatedFieldProcessingService { yield Futures.transform(relationService.findByQuery(tenantId, configuration.toEntityRelationsQuery(entityId)), configuration::resolveEntityIds, calculatedFieldCallbackExecutor); } - case CURRENT_CUSTOMER -> throw new UnsupportedOperationException(); // fixme implement + }; + } + + @Nullable + private EntityId resolveOwnerArgument(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getRefDynamicSourceConfiguration().getType()) { + case CURRENT_CUSTOMER -> { + EntityId ownerId = ownerService.getOwner(tenantId, entityId); + if (ownerId.getEntityType() == EntityType.TENANT) { + // todo: if inherit is true - use customer id + // fixme: WTF do we need it at all? + yield null; + } else { + yield ownerId; + } + } + default -> throw new UnsupportedOperationException(); }; } @@ -187,8 +222,7 @@ public abstract class AbstractCalculatedFieldProcessingService { argument.getRefEntityKey().getKey() ); return Futures.transform(attributesFuture, resultOpt -> - Map.entry(entityId, resultOpt.orElseGet(() -> - new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), + Map.entry(entityId, resultOpt.orElseGet(() -> createDefaultAttributeEntry(argument, System.currentTimeMillis()))), calculatedFieldCallbackExecutor ); }).collect(Collectors.toList()); @@ -200,6 +234,9 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + if (entityId == null) { + return Futures.immediateFuture(transformSingleValueArgument(Optional.empty())); + } return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument, startTs); case ATTRIBUTE -> fetchAttribute(tenantId, entityId, argument, startTs); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 8dd3491942..cc77913f4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; +import java.util.Set; import java.util.function.Predicate; public interface CalculatedFieldCache { @@ -47,4 +48,14 @@ public interface CalculatedFieldCache { EntityId getProfileId(TenantId tenantId, EntityId entityId); + Set getDynamicEntities(TenantId tenantId, EntityId entityId); + + void updateOwnerEntity(TenantId tenantId, EntityId entityId); + + void addOwnerEntity(TenantId tenantId, EntityId entityId); + + void evictEntity(EntityId entityId); + + void evictOwner(EntityId owner); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 0ef62c3568..36210d7302 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -23,6 +23,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -40,6 +41,7 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -59,6 +61,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final TbDeviceProfileCache deviceProfileCache; @Lazy private final ActorSystemContext systemContext; + private final OwnerService ownerService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -66,6 +69,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); + @Value("${queue.calculated_fields.init_fetch_pack_size:50000}") @Getter private int initFetchPackSize; @@ -220,6 +225,44 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { }; } + @Override + public Set getDynamicEntities(TenantId tenantId, EntityId entityId) { + if (entityId != null && entityId.getEntityType().isOneOf(EntityType.CUSTOMER, EntityType.TENANT)) { + return getOwnedEntities(tenantId, entityId); + } + return Collections.emptySet(); + } + + @Override + public void addOwnerEntity(TenantId tenantId, EntityId entityId) { + EntityId owner = ownerService.getOwner(tenantId, entityId); + getOwnedEntities(tenantId, owner).add(entityId); + } + + @Override + public void updateOwnerEntity(TenantId tenantId, EntityId entityId) { + evictEntity(entityId); + addOwnerEntity(tenantId, entityId); + } + + @Override + public void evictEntity(EntityId entityId) { + ownerEntities.values().forEach(entities -> entities.remove(entityId)); + } + + @Override + public void evictOwner(EntityId owner) { + ownerEntities.remove(owner); + } + + private Set getOwnedEntities(TenantId tenantId, EntityId ownerId) { + return ownerEntities.computeIfAbsent(ownerId, owner -> { + Set entities = ConcurrentHashMap.newKeySet(); + entities.addAll(ownerService.getOwnedEntities(tenantId, ownerId)); + return entities; + }); + } + private Lock getFetchLock(CalculatedFieldId id) { return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index f5c39ed288..9b2964a736 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -69,9 +69,10 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF TimeseriesService timeseriesService, ApiLimitService apiLimitService, RelationService relationService, + OwnerService ownerService, TbClusterService clusterService, PartitionService partitionService) { - super(attributesService, timeseriesService, apiLimitService, relationService); + super(attributesService, timeseriesService, apiLimitService, relationService, ownerService); this.clusterService = clusterService; this.partitionService = partitionService; } @@ -83,26 +84,26 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF @Override public ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchArguments(ctx, entityId); + return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { - // only geofencing calculated fields supports dynamic arguments scheduled updates + // only scheduledSupported CF instances supports dynamic arguments scheduled updates if (!ctx.getCalculatedField().getType().equals(CalculatedFieldType.GEOFENCING)) { return Map.of(); } - return resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true)); + return resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); } @Override public Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments) { Map> argFutures = new HashMap<>(); for (var entry : arguments.entrySet()) { - if (entry.getValue().hasDynamicSource()) { + if (entry.getValue().hasRelationQuerySource()) { continue; } - var argEntityId = resolveEntityId(entityId, entry.getValue()); + var argEntityId = resolveEntityId(tenantId, entityId, entry.getValue()); var argValueFuture = fetchArgumentValue(tenantId, argEntityId, entry.getValue(), System.currentTimeMillis()); argFutures.put(entry.getKey(), argValueFuture); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a3e50812fa..ab06349e3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -85,6 +85,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), + cf -> cf.dynamicSourceMatches(request.getEntries()), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -102,6 +103,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), + cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -118,6 +120,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), + cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -128,16 +131,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + cf -> cf.matchesDynamicSourceKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, - Predicate mainEntityFilter, Predicate linkedEntityFilter, + Predicate mainEntityFilter, + Predicate linkedEntityFilter, + Predicate dynamicSourceFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -148,7 +154,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { if (!supportedReferencedEntities.contains(entityId.getEntityType())) { return false; } @@ -165,6 +171,16 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } + for (EntityId dynamicEntity : calculatedFieldCache.getDynamicEntities(tenantId, entityId)) { + if (calculatedFieldCache.getCalculatedFieldCtxsByEntityId(dynamicEntity).stream().anyMatch(dynamicSourceFilter)) { + return true; + } + EntityId dynamicEntityProfileId = calculatedFieldCache.getProfileId(tenantId, dynamicEntity); + if (calculatedFieldCache.getCalculatedFieldCtxsByEntityId(dynamicEntityProfileId).stream().anyMatch(dynamicSourceFilter)) { + return true; + } + } + return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java new file mode 100644 index 0000000000..84c46715e7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceInfoFilter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; + +import java.util.HashSet; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class OwnerService { + + private final DeviceService deviceService; + private final AssetService assetService; + + public EntityId getOwner(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId).getOwnerId(); + case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId).getOwnerId(); + default -> throw new UnsupportedOperationException(); + }; + } + + public Set getOwnedEntities(TenantId tenantId, EntityId ownerId) { + Set ownerEntities = new HashSet<>(); + if (EntityType.CUSTOMER.equals(ownerId.getEntityType())) { + PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByFilter(DeviceInfoFilter.builder().tenantId(tenantId).customerId((CustomerId) ownerId).build(), pageLink), 1000); + deviceIdInfos.forEach(deviceInfo -> ownerEntities.add(deviceInfo.getId())); + PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantIdAndCustomerId(tenantId, (CustomerId) ownerId, pageLink), 1000); + assets.forEach(asset -> ownerEntities.add(asset.getId())); + } else if (EntityType.TENANT.equals(ownerId.getEntityType())) { + PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByFilter(DeviceInfoFilter.builder().tenantId((TenantId) ownerId).customerId(new CustomerId(CustomerId.NULL_UUID)).build(), pageLink), 1000); + deviceIdInfos.forEach(deviceInfo -> ownerEntities.add(deviceInfo.getId())); + PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantIdAndCustomerId((TenantId) ownerId, new CustomerId(CustomerId.NULL_UUID), pageLink), 1000); + assets.forEach(asset -> ownerEntities.add(asset.getId())); + } + return ownerEntities; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 13f5c8e6c7..5c288173de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -73,6 +73,7 @@ public class CalculatedFieldCtx { private final Map arguments; private final Map mainEntityArguments; private final Map> linkedEntityArguments; + private final Map dynamicEntityArguments; private final List argNames; private Output output; private String expression; @@ -93,7 +94,7 @@ public class CalculatedFieldCtx { private long maxSingleValueArgumentSize; private List mainEntityGeofencingArgumentNames; - private List linkedEntityGeofencingArgumentNames; + private List linkedEntityAndCurrentOwnerGeofencingArgumentNames; public CalculatedFieldCtx(CalculatedField calculatedField, ActorSystemContext systemContext) { @@ -106,19 +107,27 @@ public class CalculatedFieldCtx { this.arguments = new HashMap<>(); this.mainEntityArguments = new HashMap<>(); this.linkedEntityArguments = new HashMap<>(); + this.dynamicEntityArguments = new HashMap<>(); this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); - this.linkedEntityGeofencingArgumentNames = new ArrayList<>(); + this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); for (Map.Entry entry : arguments.entrySet()) { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); - if (refId == null && entry.getValue().hasDynamicSource()) { - continue; - } - if (refId == null || refId.equals(calculatedField.getEntityId())) { + if (refId == null) { + // TODO: no matchers for this type of source exists yet, so no reason to add to dynamicEntityArguments map. + if (entry.getValue().hasRelationQuerySource()) { + continue; + } + if (entry.getValue().hasOwnerSource()) { + dynamicEntityArguments.put(refKey, entry.getKey()); + } else { + mainEntityArguments.put(refKey, entry.getKey()); + } + } else if (refId.equals(calculatedField.getEntityId())) { mainEntityArguments.put(refKey, entry.getKey()); } else { linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); @@ -135,8 +144,8 @@ public class CalculatedFieldCtx { mainEntityGeofencingArgumentNames.add(zoneGroupName); return; } - if (config.isLinkedCfEntitySource(entityId)) { - linkedEntityGeofencingArgumentNames.add(zoneGroupName); + if (config.isLinkedCfEntitySource(entityId) || config.hasCurrentOwnerSource()) { + linkedEntityAndCurrentOwnerGeofencingArgumentNames.add(zoneGroupName); } }); } @@ -292,6 +301,14 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeries(map, values); } + public boolean dynamicSourceMatches(List values) { + return matchesTimeSeries(dynamicEntityArguments, values); + } + + public boolean dynamicSourceMatches(List values, AttributeScope scope) { + return matchesAttributes(dynamicEntityArguments, values, scope); + } + private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { if (argMap.isEmpty() || values.isEmpty()) { return false; @@ -335,6 +352,14 @@ public class CalculatedFieldCtx { return matchesTimeSeriesKeys(mainEntityArguments, keys); } + public boolean matchesDynamicSourceKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(dynamicEntityArguments, keys, scope); + } + + public boolean matchesDynamicSourceKeys(List keys) { + return matchesTimeSeriesKeys(dynamicEntityArguments, keys); + } + private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { if (argMap.isEmpty() || keys.isEmpty()) { return false; @@ -381,6 +406,25 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeriesKeys(map, keys); } + public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return dynamicSourceMatches(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return dynamicSourceMatches(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); + } else { + return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() @@ -400,6 +444,18 @@ public class CalculatedFieldCtx { } } + public Map getLinkedAndDynamicArgs(EntityId entityId) { + var argNames = new HashMap(); + var linkedArgNames = linkedEntityArguments.get(entityId); + if (linkedArgNames != null && !linkedArgNames.isEmpty()) { + argNames.putAll(linkedArgNames); + } + if (dynamicEntityArguments != null && !dynamicEntityArguments.isEmpty()) { + argNames.putAll(dynamicEntityArguments); + } + return argNames; + } + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -457,6 +513,10 @@ public class CalculatedFieldCtx { return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; } + public boolean hasCurrentOwnerSourceArguments() { + return !dynamicEntityArguments.isEmpty(); + } + @Override public String toString() { return "CalculatedFieldCtx{" + diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 265f14c4e2..418cf2c362 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -668,6 +668,7 @@ public class DefaultTbClusterService implements TbClusterService { } msg.event(ComponentLifecycleEvent.UPDATED) .oldProfileId(old.getDeviceProfileId()) + .ownerChanged(!entity.getOwnerId().equals(old.getOwnerId())) .oldName(old.getName()); } broadcast(msg.build()); @@ -688,6 +689,7 @@ public class DefaultTbClusterService implements TbClusterService { } else { msg.event(ComponentLifecycleEvent.UPDATED) .oldProfileId(old.getAssetProfileId()) + .ownerChanged(!entity.getOwnerId().equals(old.getOwnerId())) .oldName(old.getName()); } broadcast(msg.build()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index 3dd6993962..b7d561f005 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -87,8 +87,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService new SimpleCalculatedFieldState(entityId); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 1bfb2bf875..581fad9a27 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -25,6 +25,7 @@ import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -40,6 +41,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CurrentCustomerDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; @@ -74,13 +76,14 @@ public class AlarmRulesTest extends AbstractControllerTest { @Autowired private EventDao eventDao; + private Device device; private DeviceId deviceId; private EventId latestEventId; @Before public void beforeEach() throws Exception { loginTenantAdmin(); - Device device = createDevice("Device A", "aaa"); + device = createDevice("Device A", "aaa"); deviceId = device.getId(); } @@ -207,6 +210,40 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + @Test + public void testCreateAlarm_currentOwnerArgument() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + + Argument temperatureThresholdArgument = new Argument(); + temperatureThresholdArgument.setRefEntityKey(new ReferencedEntityKey("temperatureThreshold", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + temperatureThresholdArgument.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + temperatureThresholdArgument.setDefaultValue("1000"); + + Map arguments = Map.of( + "temperature", temperatureArgument, + "temperatureThreshold", temperatureThresholdArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= temperatureThreshold;", null, null) + ); + + device.setCustomerId(customerId); + device = doPost("/api/device", device, Device.class); + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"temperatureThreshold\":50}"); + + postTelemetry(deviceId, "{\"temperature\":51}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { TbAlarmResult alarmResult = getLatestAlarmResult(calculatedField.getId()); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 28e9d5bb1a..6db510192a 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; -import org.junit.jupiter.api.BeforeEach; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; @@ -67,11 +66,6 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes public static final int TIMEOUT = 60; public static final int POLL_INTERVAL = 1; - @BeforeEach - void setUp() throws Exception { - loginTenantAdmin(); - } - @Test public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 8e7d2bb5ff..d2ac4741f4 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -77,6 +77,7 @@ import org.thingsboard.server.actors.device.DeviceActorMessageProcessor; import org.thingsboard.server.actors.device.SessionInfo; import org.thingsboard.server.actors.device.ToDeviceRpcRequestMetadata; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -1317,8 +1318,13 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } protected void postTelemetry(EntityId entityId, String payload) throws Exception { - doPost("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + - "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload)).andExpect(status().isOk()); + doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); + } + + protected void postAttributes(EntityId entityId, AttributeScope scope, String payload) throws Exception { + doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + + "/attributes/" + scope, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); } protected CalculatedField saveCalculatedField(CalculatedField calculatedField) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 57a0d24466..e170dbc467 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; @@ -142,6 +143,11 @@ public class Device extends BaseDataWithAdditionalInfo implements HasL this.customerId = customerId; } + @JsonIgnore + public EntityId getOwnerId() { + return customerId != null && !customerId.isNullUid() ? customerId : tenantId; + } + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "Unique Device Name in scope of Tenant", example = "A4B72CCDFF33") @Override public String getName() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java index 22934de813..1fb49a46f7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -34,21 +34,23 @@ public class ProfileEntityIdInfo implements Serializable, HasTenantId { private static final long serialVersionUID = 8532058281983868003L; private final TenantId tenantId; + private final EntityId ownerId; private final EntityId profileId; private final EntityId entityId; - private ProfileEntityIdInfo(UUID tenantId, EntityId profileId, EntityId entityId) { + private ProfileEntityIdInfo(UUID tenantId, EntityId ownerId, EntityId profileId, EntityId entityId) { this.tenantId = TenantId.fromUUID(tenantId); + this.ownerId = ownerId; this.profileId = profileId; this.entityId = entityId; } - public static ProfileEntityIdInfo create(UUID tenantId, DeviceProfileId profileId, DeviceId entityId) { - return new ProfileEntityIdInfo(tenantId, profileId, entityId); + public static ProfileEntityIdInfo create(UUID tenantId, EntityId ownerId, DeviceProfileId profileId, DeviceId entityId) { + return new ProfileEntityIdInfo(tenantId, ownerId, profileId, entityId); } - public static ProfileEntityIdInfo create(UUID tenantId, AssetProfileId profileId, AssetId entityId) { - return new ProfileEntityIdInfo(tenantId, profileId, entityId); + public static ProfileEntityIdInfo create(UUID tenantId, EntityId ownerId, AssetProfileId profileId, AssetId entityId) { + return new ProfileEntityIdInfo(tenantId, ownerId, profileId, entityId); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index e732049118..a34b58a4da 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.asset; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @@ -125,6 +127,11 @@ public class Asset extends BaseDataWithAdditionalInfo implements HasLab this.customerId = customerId; } + @JsonIgnore + public EntityId getOwnerId() { + return customerId != null && !customerId.isNullUid() ? customerId : tenantId; + } + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "Unique Asset Name in scope of Tenant", example = "Empire State Building") @Override public String getName() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 52935c3411..0aad0737b8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -37,4 +37,12 @@ public class Argument { return refDynamicSourceConfiguration != null; } + public boolean hasRelationQuerySource() { + return hasDynamicSource() && CFArgumentDynamicSourceType.RELATION_QUERY.equals(refDynamicSourceConfiguration.getType()); + } + + public boolean hasOwnerSource() { + return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_CUSTOMER; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 535febf3a0..b72cdad60a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -31,8 +31,8 @@ public abstract class BaseCalculatedFieldConfiguration implements ExpressionBase if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); } - if (arguments.values().stream().anyMatch(Argument::hasDynamicSource)) { - throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support dynamic source configuration!"); + if (arguments.values().stream().anyMatch(Argument::hasRelationQuerySource)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support relation query source configuration!"); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java index dc331f5876..ff251a3ad3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java @@ -63,7 +63,7 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal @Override public boolean isScheduledUpdateEnabled() { - return scheduledUpdateInterval > 0 && zoneGroups.values().stream().anyMatch(ZoneGroupConfiguration::hasDynamicSource); + return scheduledUpdateInterval > 0 && zoneGroups.values().stream().anyMatch(ZoneGroupConfiguration::hasRelationQuerySource); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java index 2feb6e49d0..775f711a5e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java @@ -54,7 +54,7 @@ public class ZoneGroupConfiguration { if (reportStrategy == null) { throw new IllegalArgumentException("Report strategy must be specified for '" + name + "' zone group!"); } - if (hasDynamicSource()) { + if (refDynamicSourceConfiguration != null) { refDynamicSourceConfiguration.validate(); } if (!createRelationsWithMatchedZones) { @@ -68,8 +68,12 @@ public class ZoneGroupConfiguration { } } - public boolean hasDynamicSource() { - return refDynamicSourceConfiguration != null; + public boolean hasRelationQuerySource() { + return toArgument().hasRelationQuerySource(); + } + + public boolean hasCurrentOwnerSource() { + return toArgument().hasOwnerSource(); } @JsonIgnore @@ -92,4 +96,5 @@ public class ZoneGroupConfiguration { argument.setRefEntityKey(new ReferencedEntityKey(perimeterKeyName, ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); return argument; } + } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java index fd59317649..260a39a8bc 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java @@ -29,10 +29,21 @@ public class ArgumentTest { } @Test - void validateShouldReturnTrueIfDynamicSourceConfigurationIsNotNull() { + void validateWhenRelationQuerySourceConfigurationIsNotNull() { var argument = new Argument(); argument.setRefDynamicSourceConfiguration(new RelationQueryDynamicSourceConfiguration()); assertThat(argument.hasDynamicSource()).isTrue(); + assertThat(argument.hasRelationQuerySource()).isTrue(); + assertThat(argument.hasOwnerSource()).isFalse(); + } + + @Test + void validateWhenCurrentCustomerSourceConfigurationIsNotNull() { + var argument = new Argument(); + argument.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + assertThat(argument.hasDynamicSource()).isTrue(); + assertThat(argument.hasOwnerSource()).isTrue(); + assertThat(argument.hasRelationQuerySource()).isFalse(); } } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java index 91a47aac57..9f2f49bc20 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java @@ -24,14 +24,12 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; @@ -112,7 +110,7 @@ public class GeofencingCalculatedFieldConfigurationTest { void scheduledUpdateDisabledWhenIntervalIsGreaterThanZeroButNoZonesWithDynamicArguments() { var cfg = new GeofencingCalculatedFieldConfiguration(); var zoneGroupConfigurationMock = mock(ZoneGroupConfiguration.class); - when(zoneGroupConfigurationMock.hasDynamicSource()).thenReturn(false); + when(zoneGroupConfigurationMock.hasRelationQuerySource()).thenReturn(false); cfg.setZoneGroups(Map.of("someGroupName", zoneGroupConfigurationMock)); cfg.setScheduledUpdateInterval(60); assertThat(cfg.isScheduledUpdateEnabled()).isFalse(); @@ -122,7 +120,7 @@ public class GeofencingCalculatedFieldConfigurationTest { void scheduledUpdateEnabledWhenIntervalIsGreaterThanZeroAndDynamicArgumentsPresent() { var cfg = new GeofencingCalculatedFieldConfiguration(); var zoneGroupConfigurationMock = mock(ZoneGroupConfiguration.class); - when(zoneGroupConfigurationMock.hasDynamicSource()).thenReturn(true); + when(zoneGroupConfigurationMock.hasRelationQuerySource()).thenReturn(true); cfg.setZoneGroups(Map.of("someGroupName", zoneGroupConfigurationMock)); cfg.setScheduledUpdateInterval(60); assertThat(cfg.isScheduledUpdateEnabled()).isTrue(); diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java index 4eb822d93c..7bb657fb33 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CurrentCustomerDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -98,19 +99,25 @@ public class ZoneGroupConfigurationTest { } @Test - void whenHasDynamicSourceCalled_shouldReturnTrueIfDynamicSourceConfigurationIsNotNull() { + void whenHasRelationQuerySourceCalled_shouldReturnTrueIfRelationQuerySourceConfigurationIsNotNull() { var zoneGroupConfiguration = new ZoneGroupConfiguration("perimeter", REPORT_TRANSITION_EVENTS_AND_PRESENCE_STATUS, false); zoneGroupConfiguration.setRefDynamicSourceConfiguration(new RelationQueryDynamicSourceConfiguration()); - assertThat(zoneGroupConfiguration.hasDynamicSource()).isTrue(); + assertThat(zoneGroupConfiguration.hasRelationQuerySource()).isTrue(); } @Test - void whenHasDynamicSourceCalled_shouldReturnTrueIfDynamicSourceConfigurationIsNull() { + void whenHasRelationQuerySourceCalled_shouldReturnFalseIfRelationQuerySourceConfigurationIsNull() { var zoneGroupConfiguration = mock(ZoneGroupConfiguration.class); assertThat(zoneGroupConfiguration.getRefDynamicSourceConfiguration()).isNull(); - assertThat(zoneGroupConfiguration.hasDynamicSource()).isFalse(); + assertThat(zoneGroupConfiguration.hasRelationQuerySource()).isFalse(); } + @Test + void whenHasRelationQuerySourceCalled_shouldReturnFalseIfCurrentCustomerSourceConfigured() { + var zoneGroupConfiguration = mock(ZoneGroupConfiguration.class); + zoneGroupConfiguration.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + assertThat(zoneGroupConfiguration.hasRelationQuerySource()).isFalse(); + } @Test void validateToArgumentsMethodCallWithoutRefEntityId() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index fca3632ee8..6451ea0a6f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -151,6 +151,7 @@ public enum MsgType { CF_ENTITY_INIT_CF_MSG, CF_ENTITY_DELETE_MSG, + CF_ARGUMENT_RESET_MSG, // Sent to reset argument; CF_DYNAMIC_ARGUMENTS_REFRESH_MSG, CF_ENTITY_DYNAMIC_ARGUMENTS_REFRESH_MSG, CF_REEVALUATE_MSG; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index d57301fd10..23b9fe08e3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -46,14 +46,15 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final String name; private final EntityId oldProfileId; private final EntityId profileId; + private final boolean ownerChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, null); + this(tenantId, entityId, event, null, null, null, null, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -61,6 +62,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.name = name; this.oldProfileId = oldProfileId; this.profileId = profileId; + this.ownerChanged = ownerChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index f5a07cf07e..26a64c7f8a 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -129,6 +129,7 @@ public class ProtoUtils { builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } + builder.setOwnerChanged(msg.isOwnerChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -165,6 +166,7 @@ public class ProtoUtils { var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } + builder.ownerChanged(proto.getOwnerChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 5a8e348f5f..fac1116a30 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1291,6 +1291,7 @@ message ComponentLifecycleMsgProto { int64 profileIdMSB = 11; int64 profileIdLSB = 12; optional string info = 13; + bool ownerChanged = 100; } message EdgeEventMsgProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 05b782c26c..2c6e9160ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -100,7 +100,7 @@ public class CalculatedFieldDataValidator extends DataValidator } Map relationQueryBasedArguments = argumentsBasedCfg.getArguments().entrySet() .stream() - .filter(entry -> entry.getValue().hasDynamicSource()) + .filter(entry -> entry.getValue().hasRelationQuerySource()) .collect(Collectors.toMap(Map.Entry::getKey, entry -> (RelationQueryDynamicSourceConfiguration) entry.getValue().getRefDynamicSourceConfiguration())); if (relationQueryBasedArguments.isEmpty()) { return; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java index ec47da3499..3057b0d30f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -23,9 +23,12 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import java.util.Map; import java.util.UUID; @Repository @@ -40,23 +43,24 @@ public class DefaultNativeAssetRepository extends AbstractNativeRepository imple @Override public PageData findProfileEntityIdInfos(Pageable pageable) { - String PROFILE_ASSET_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; - return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, row -> { - AssetId id = new AssetId((UUID) row.get("id")); - AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); - var tenantIdObj = row.get("tenantId"); - return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); - }); + String PROFILE_ASSET_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, DefaultNativeAssetRepository::toInfo); } @Override public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { - String PROFILE_ASSET_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); - return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, row -> { - AssetId id = new AssetId((UUID) row.get("id")); - AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); - var tenantIdObj = row.get("tenantId"); - return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); - }); + String PROFILE_ASSET_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, customer_id as customerId, asset_profile_id as profileId, id as id FROM asset WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, DefaultNativeAssetRepository::toInfo); } + + private static ProfileEntityIdInfo toInfo(Map row) { + var tenantIdObj = row.get("tenantId"); + UUID tenantId = tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(); + AssetId id = new AssetId((UUID) row.get("id")); + CustomerId customerId = new CustomerId((UUID) row.get("customerId")); + EntityId ownerId = !customerId.isNullUid() ? customerId : TenantId.fromUUID(tenantId); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + return ProfileEntityIdInfo.create(tenantId, ownerId, profileId, id); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 78ee2795b0..49062f829f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -22,11 +22,14 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import java.util.Map; import java.util.UUID; @Repository @@ -52,24 +55,24 @@ public class DefaultNativeDeviceRepository extends AbstractNativeRepository impl @Override public PageData findProfileEntityIdInfos(Pageable pageable) { - String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; - return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { - DeviceId id = new DeviceId((UUID) row.get("id")); - DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); - var tenantIdObj = row.get("tenantId"); - return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); - }); + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, DefaultNativeDeviceRepository::toInfo); } @Override public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { - String PROFILE_DEVICE_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); - return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { - DeviceId id = new DeviceId((UUID) row.get("id")); - DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); - var tenantIdObj = row.get("tenantId"); - return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); - }); + String PROFILE_DEVICE_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, customer_id as customerId, device_profile_id as profileId, id as id FROM device WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, DefaultNativeDeviceRepository::toInfo); + } + + private static ProfileEntityIdInfo toInfo(Map row) { + var tenantIdObj = row.get("tenantId"); + UUID tenantId = tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(); + DeviceId id = new DeviceId((UUID) row.get("id")); + CustomerId customerId = new CustomerId((UUID) row.get("customerId")); + EntityId ownerId = !customerId.isNullUid() ? customerId : TenantId.fromUUID(tenantId); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + return ProfileEntityIdInfo.create(tenantId, ownerId, profileId, id); } } From 56059ade0805419f3a77cf9faedabcae8a8a737a Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Tue, 30 Sep 2025 13:06:20 +0300 Subject: [PATCH 008/122] Rename "Current customer" source to "Current owner" --- ...tractCalculatedFieldProcessingService.java | 34 +-- .../thingsboard/server/cf/AlarmRulesTest.java | 4 +- .../cf/CalculatedFieldCurrentOwnerTest.java | 200 ++++++++++++++++++ .../data/cf/configuration/Argument.java | 2 +- .../CFArgumentDynamicSourceType.java | 2 +- .../CfArgumentDynamicSourceConfiguration.java | 2 +- ...rrentOwnerDynamicSourceConfiguration.java} | 6 +- .../data/cf/configuration/ArgumentTest.java | 4 +- .../ZoneGroupConfigurationTest.java | 6 +- 9 files changed, 216 insertions(+), 44 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/cf/CalculatedFieldCurrentOwnerTest.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/{CurrentCustomerDynamicSourceConfiguration.java => CurrentOwnerDynamicSourceConfiguration.java} (78%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 343db5286f..90d3913263 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -19,13 +19,11 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationQueryDynamicSourceConfiguration; @@ -47,7 +45,6 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +111,7 @@ public abstract class AbstractCalculatedFieldProcessingService { if (!argument.hasOwnerSource()) { return entityId; } - return resolveOwnerArgument(tenantId, entityId, argument); + return resolveOwnerArgument(tenantId, entityId); } protected Map resolveArgumentFutures(Map> argFutures) { @@ -166,14 +163,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } var refDynamicSourceConfiguration = value.getRefDynamicSourceConfiguration(); return switch (refDynamicSourceConfiguration.getType()) { - case CURRENT_CUSTOMER -> { - EntityId resolved = resolveOwnerArgument(tenantId, entityId, value); - if (resolved != null) { - yield Futures.immediateFuture(List.of(resolved)); - } else { - yield Futures.immediateFuture(Collections.emptyList()); - } - } + case CURRENT_OWNER -> Futures.immediateFuture(List.of(resolveOwnerArgument(tenantId, entityId))); case RELATION_QUERY -> { var configuration = (RelationQueryDynamicSourceConfiguration) refDynamicSourceConfiguration; if (configuration.isSimpleRelation()) { @@ -192,21 +182,8 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - @Nullable - private EntityId resolveOwnerArgument(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getRefDynamicSourceConfiguration().getType()) { - case CURRENT_CUSTOMER -> { - EntityId ownerId = ownerService.getOwner(tenantId, entityId); - if (ownerId.getEntityType() == EntityType.TENANT) { - // todo: if inherit is true - use customer id - // fixme: WTF do we need it at all? - yield null; - } else { - yield ownerId; - } - } - default -> throw new UnsupportedOperationException(); - }; + private EntityId resolveOwnerArgument(TenantId tenantId, EntityId entityId) { + return ownerService.getOwner(tenantId, entityId); } private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { @@ -234,9 +211,6 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { - if (entityId == null) { - return Futures.immediateFuture(transformSingleValueArgument(Optional.empty())); - } return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument, startTs); case ATTRIBUTE -> fetchAttribute(tenantId, entityId, argument, startTs); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 581fad9a27..9087d9217a 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -41,7 +41,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CurrentCustomerDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CurrentOwnerDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; @@ -218,7 +218,7 @@ public class AlarmRulesTest extends AbstractControllerTest { Argument temperatureThresholdArgument = new Argument(); temperatureThresholdArgument.setRefEntityKey(new ReferencedEntityKey("temperatureThreshold", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); - temperatureThresholdArgument.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + temperatureThresholdArgument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); temperatureThresholdArgument.setDefaultValue("1000"); Map arguments = Map.of( diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldCurrentOwnerTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldCurrentOwnerTest.java new file mode 100644 index 0000000000..d2f9621064 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldCurrentOwnerTest.java @@ -0,0 +1,200 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CurrentOwnerDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldCurrentOwnerTest extends AbstractControllerTest { + + public static final int TIMEOUT = 60; + public static final int POLL_INTERVAL = 1; + + @Test + public void testCreateCFWithCurrentOwner() throws Exception { + loginTenantAdmin(); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":5}"); + + Device testDevice = createDevice("Test device", "1234567890"); + + doPost("/api/customer/" + customerId.getId() + "/device/" + testDevice.getId().getId()).andExpect(status().isOk()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", buildCalculatedField(testDevice.getId()), CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "result"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("result").get(0).get("value").asText()).isEqualTo("105"); + }); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":10}"); + + await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "result"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("result").get(0).get("value").asText()).isEqualTo("110"); + }); + } + + @Test + public void testChangeOwner() throws Exception { + loginSysAdmin(); + + postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":50}"); + + loginTenantAdmin(); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":5}"); + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/customer/" + customerId.getId() + "/device/" + testDevice.getId().getId()).andExpect(status().isOk()); + + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", buildCalculatedField(testDevice.getId()), CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "result"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("result").get(0).get("value").asText()).isEqualTo("105"); + }); + + doDelete("/api/customer/device/" + testDevice.getId().getId()).andExpect(status().isOk()); + + await().alias("change owner -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "result"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("result").get(0).get("value").asText()).isEqualTo("150"); + }); + } + + @Test + public void testCreateCFWithCurrentOwnerWhenEntityIsProfile() throws Exception { + loginSysAdmin(); + + postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":50}"); + + loginTenantAdmin(); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"attrKey\":5}"); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/customer/" + customerId.getId() + "/asset/" + asset1.getId().getId()).andExpect(status().isOk()); + + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); // owner - TENANT + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", buildCalculatedField(assetProfile.getId()), CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ObjectNode result1 = getLatestTelemetry(asset1.getId(), "result"); + assertThat(result1).isNotNull(); + assertThat(result1.get("result").get(0).get("value").asText()).isEqualTo("105"); + + // result of asset 2 + ObjectNode result2 = getLatestTelemetry(asset2.getId(), "result"); + assertThat(result2).isNotNull(); + assertThat(result2.get("result").get(0).get("value").asText()).isEqualTo("150"); + }); + + doPost("/api/customer/" + customerId.getId() + "/asset/" + asset2.getId().getId()).andExpect(status().isOk()); + + await().alias("change asset2 owner -> recalculate state for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 2 + ObjectNode result2 = getLatestTelemetry(asset2.getId(), "result"); + assertThat(result2).isNotNull(); + assertThat(result2.get("result").get(0).get("value").asText()).isEqualTo("105"); + }); + } + + private CalculatedField buildCalculatedField(EntityId entityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("attrKey", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument.setRefEntityKey(refEntityKey); + argument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + + config.setArguments(Map.of("a", argument)); + + config.setExpression("a + 100"); + + Output output = new Output(); + output.setName("result"); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + return calculatedField; + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 0aad0737b8..04d926dc2d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -42,7 +42,7 @@ public class Argument { } public boolean hasOwnerSource() { - return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_CUSTOMER; + return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_OWNER; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java index e8ef6c7835..3751694eb8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CFArgumentDynamicSourceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.cf.configuration; public enum CFArgumentDynamicSourceType { - CURRENT_CUSTOMER, + CURRENT_OWNER, RELATION_QUERY } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java index c16d8abfcc..639bd18b46 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CfArgumentDynamicSourceConfiguration.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; ) @JsonSubTypes({ @JsonSubTypes.Type(value = RelationQueryDynamicSourceConfiguration.class, name = "RELATION_QUERY"), - @JsonSubTypes.Type(value = CurrentCustomerDynamicSourceConfiguration.class, name = "CURRENT_CUSTOMER") + @JsonSubTypes.Type(value = CurrentOwnerDynamicSourceConfiguration.class, name = "CURRENT_OWNER") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CfArgumentDynamicSourceConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentOwnerDynamicSourceConfiguration.java similarity index 78% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentOwnerDynamicSourceConfiguration.java index 8ede2c28df..be9a519f1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentCustomerDynamicSourceConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CurrentOwnerDynamicSourceConfiguration.java @@ -18,13 +18,11 @@ package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; @Data -public class CurrentCustomerDynamicSourceConfiguration implements CfArgumentDynamicSourceConfiguration { - - private boolean inherit; // TODO: implement +public class CurrentOwnerDynamicSourceConfiguration implements CfArgumentDynamicSourceConfiguration { @Override public CFArgumentDynamicSourceType getType() { - return CFArgumentDynamicSourceType.CURRENT_CUSTOMER; + return CFArgumentDynamicSourceType.CURRENT_OWNER; } } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java index 260a39a8bc..6ac4e63e5f 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ArgumentTest.java @@ -38,9 +38,9 @@ public class ArgumentTest { } @Test - void validateWhenCurrentCustomerSourceConfigurationIsNotNull() { + void validateWhenCurrentOwnerSourceConfigurationIsNotNull() { var argument = new Argument(); - argument.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + argument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); assertThat(argument.hasDynamicSource()).isTrue(); assertThat(argument.hasOwnerSource()).isTrue(); assertThat(argument.hasRelationQuerySource()).isFalse(); diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java index 7bb657fb33..c2dbc17f57 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CurrentCustomerDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CurrentOwnerDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -113,9 +113,9 @@ public class ZoneGroupConfigurationTest { } @Test - void whenHasRelationQuerySourceCalled_shouldReturnFalseIfCurrentCustomerSourceConfigured() { + void whenHasRelationQuerySourceCalled_shouldReturnFalseIfCurrentOwnerSourceConfigured() { var zoneGroupConfiguration = mock(ZoneGroupConfiguration.class); - zoneGroupConfiguration.setRefDynamicSourceConfiguration(new CurrentCustomerDynamicSourceConfiguration()); + zoneGroupConfiguration.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); assertThat(zoneGroupConfiguration.hasRelationQuerySource()).isFalse(); } From 151dfe4c835901e802a2fec47f8c027cd8f11972 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 2 Oct 2025 11:22:53 +0300 Subject: [PATCH 009/122] Alarm rules CF: support for filter predicates --- .../alarm/AlarmCalculatedFieldState.java | 168 ++++++++++++++++-- .../cf/ctx/state/alarm/AlarmRuleState.java | 22 +-- .../thingsboard/server/cf/AlarmRulesTest.java | 84 ++++++++- .../expression/AlarmConditionFilter.java | 37 ++++ .../expression/ComplexOperation.java | 21 +++ .../SimpleAlarmConditionExpression.java | 11 +- .../predicate/BooleanFilterPredicate.java | 37 ++++ .../predicate/ComplexFilterPredicate.java | 34 ++++ .../predicate/FilterPredicateType.java | 23 +++ .../predicate/KeyFilterPredicate.java | 36 ++++ .../predicate/NumericFilterPredicate.java | 41 +++++ .../predicate/SimpleKeyFilterPredicate.java | 24 +++ .../predicate/StringFilterPredicate.java | 43 +++++ 13 files changed, 538 insertions(+), 43 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/ComplexOperation.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/ComplexFilterPredicate.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/FilterPredicateType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/KeyFilterPredicate.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/SimpleKeyFilterPredicate.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index bc40af2568..9b3f28cf3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.KvUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; @@ -33,13 +34,23 @@ import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionFilter; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.ComplexOperation; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.SimpleAlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.BooleanFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.ComplexFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.KeyFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.AlarmCalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -52,6 +63,9 @@ import java.util.Map; import java.util.TreeMap; import java.util.function.Function; +import static org.thingsboard.server.common.data.StringUtils.equalsAny; +import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; + @EqualsAndHashCode(callSuper = true) @Slf4j public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @@ -131,20 +145,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { .build()); } - @SneakyThrows - public boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { - if (expression instanceof TbelAlarmConditionExpression tbelExpression) { - Object result = ctx.evaluateTbelExpression(tbelExpression.getExpression(), this).get(); - if (result instanceof Boolean booleanResult) { - return booleanResult; - } else { - throw new IllegalStateException("Condition expression returned non-boolean value: '" + result + "'"); - } - } else { - throw new UnsupportedOperationException("Simple expressions not supported"); - } - } - public void processAlarmAction(Alarm alarm, ActionType action) { switch (action) { case ALARM_ACK -> processAlarmAck(alarm); @@ -319,6 +319,146 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { return alarmDetails; } + @SneakyThrows + public boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { + if (expression instanceof TbelAlarmConditionExpression tbelExpression) { + Object result = ctx.evaluateTbelExpression(tbelExpression.getExpression(), this).get(); + if (result instanceof Boolean booleanResult) { + return booleanResult; + } else { + throw new IllegalStateException("Condition expression returned non-boolean value: '" + result + "'"); + } + } else { + SimpleAlarmConditionExpression simpleExpression = (SimpleAlarmConditionExpression) expression; + ComplexOperation operation = simpleExpression.getOperation(); + if (operation == null) { + operation = ComplexOperation.AND; + } + return switch (operation) { + case OR -> { + for (AlarmConditionFilter filter : simpleExpression.getFilters()) { + SingleValueArgumentEntry argument = getArgument(filter.getArgument()); + if (eval(argument, filter.getPredicate())) { + yield true; + } + } + yield false; + } + case AND -> { + for (AlarmConditionFilter filter : simpleExpression.getFilters()) { + SingleValueArgumentEntry argument = getArgument(filter.getArgument()); + if (!eval(argument, filter.getPredicate())) { + yield false; + } + } + yield true; + } + }; + } + } + + private boolean eval(SingleValueArgumentEntry argument, KeyFilterPredicate predicate) { + return switch (predicate.getType()) { + case STRING -> evalStrPredicate(argument, (StringFilterPredicate) predicate); + case NUMERIC -> evalNumPredicate(argument, (NumericFilterPredicate) predicate); + case BOOLEAN -> evalBooleanPredicate(argument, (BooleanFilterPredicate) predicate); + case COMPLEX -> evalComplexPredicate(argument, (ComplexFilterPredicate) predicate); + }; + } + + private boolean evalComplexPredicate(SingleValueArgumentEntry argument, ComplexFilterPredicate complexPredicate) { + return switch (complexPredicate.getOperation()) { + case OR -> { + for (KeyFilterPredicate predicate : complexPredicate.getPredicates()) { + if (eval(argument, predicate)) { + yield true; + } + } + yield false; + } + case AND -> { + for (KeyFilterPredicate predicate : complexPredicate.getPredicates()) { + if (!eval(argument, predicate)) { + yield false; + } + } + yield true; + } + }; + } + + private boolean evalBooleanPredicate(SingleValueArgumentEntry argument, BooleanFilterPredicate predicate) { + Boolean value = KvUtil.getBoolValue(argument.getKvEntryValue()); + if (value == null) { + return false; + } + Boolean predicateValue = resolveValue(predicate.getValue(), KvUtil::getBoolValue); + if (predicateValue == null) { + return false; + } + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + }; + } + + private boolean evalNumPredicate(SingleValueArgumentEntry argument, NumericFilterPredicate predicate) { + Double value = KvUtil.getDoubleValue(argument.getKvEntryValue()); + if (value == null) { + return false; + } + Double predicateValue = resolveValue(predicate.getValue(), KvUtil::getDoubleValue); + if (predicateValue == null) { + return false; + } + return switch (predicate.getOperation()) { + case NOT_EQUAL -> !value.equals(predicateValue); + case EQUAL -> value.equals(predicateValue); + case GREATER -> value > predicateValue; + case GREATER_OR_EQUAL -> value >= predicateValue; + case LESS -> value < predicateValue; + case LESS_OR_EQUAL -> value <= predicateValue; + }; + } + + private boolean evalStrPredicate(SingleValueArgumentEntry argument, StringFilterPredicate predicate) { + String value = KvUtil.getStringValue(argument.getKvEntryValue()); + if (value == null) { + return false; + } + String predicateValue = resolveValue(predicate.getValue(), KvUtil::getStringValue); + if (predicateValue == null) { + return false; + } + if (predicate.isIgnoreCase()) { + value = value.toLowerCase(); + predicateValue = predicateValue.toLowerCase(); + } + return switch (predicate.getOperation()) { + case CONTAINS -> value.contains(predicateValue); + case EQUAL -> value.equals(predicateValue); + case STARTS_WITH -> value.startsWith(predicateValue); + case ENDS_WITH -> value.endsWith(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case NOT_CONTAINS -> !value.contains(predicateValue); + case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + }; + } + + protected T resolveValue(AlarmConditionValue conditionValue, Function mapper) { + T value = conditionValue.getStaticValue(); + if (value == null) { + String argument = conditionValue.getDynamicValueArgument(); + SingleValueArgumentEntry entry = getArgument(argument); + value = mapper.apply(entry.getKvEntryValue()); + if (value == null) { + throw new IllegalArgumentException("No value found for argument " + argument); + } + } + return value; + } + protected SingleValueArgumentEntry getArgument(String key) { SingleValueArgumentEntry entry = (SingleValueArgumentEntry) arguments.get(key); if (entry == null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index fc209110fc..0e5459af5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -31,16 +31,13 @@ import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSch import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeSchedule; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeScheduleItem; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.SpecificTimeSchedule; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.tools.SchedulerUtils; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Optional; -import java.util.function.Function; @Data @Slf4j @@ -132,7 +129,7 @@ public class AlarmRuleState { if (condition.getSchedule() == null) { return true; } - AlarmSchedule schedule = getValue(condition.getSchedule(), entry -> Optional.ofNullable(KvUtil.getStringValue(entry)) + AlarmSchedule schedule = state.resolveValue(condition.getSchedule(), entry -> Optional.ofNullable(KvUtil.getStringValue(entry)) .map(str -> JsonConverter.parse(str, AlarmSchedule.class)) .orElse(null)); return switch (schedule.getType()) { @@ -198,31 +195,18 @@ public class AlarmRuleState { } private Integer getIntValue(AlarmConditionValue value) { - return getValue(value, entry -> Optional.ofNullable(KvUtil.getLongValue(entry)).map(Long::intValue).orElse(null)); + return state.resolveValue(value, entry -> Optional.ofNullable(KvUtil.getLongValue(entry)).map(Long::intValue).orElse(null)); } private long getRequiredDurationInMs() { DurationAlarmCondition durationCondition = (DurationAlarmCondition) condition; - return durationCondition.getUnit().toMillis(getValue(durationCondition.getValue(), KvUtil::getLongValue)); + return durationCondition.getUnit().toMillis(state.resolveValue(durationCondition.getValue(), KvUtil::getLongValue)); } private boolean eval(AlarmConditionExpression expression, CalculatedFieldCtx ctx) { return state.eval(expression, ctx); } - private T getValue(AlarmConditionValue conditionValue, Function mapper) { - T value = conditionValue.getStaticValue(); - if (value == null) { - String argument = conditionValue.getDynamicValueArgument(); - SingleValueArgumentEntry entry = state.getArgument(argument); - value = mapper.apply(entry.getKvEntryValue()); - if (value == null) { - throw new IllegalArgumentException("No value found for argument " + argument); - } - } - return value; - } - public void setAlarmRule(AlarmRule alarmRule) { this.alarmRule = alarmRule; this.condition = alarmRule.getCondition(); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 9087d9217a..1a28a5aef8 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.cf; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.Before; @@ -35,7 +36,13 @@ import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionVal import org.thingsboard.server.common.data.alarm.rule.condition.DurationAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.RepeatingAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.SimpleAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionFilter; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.ComplexOperation; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.SimpleAlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate.NumericOperation; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; @@ -134,6 +141,41 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + @Test + public void testCreateAlarm_simpleConditionExpression() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + SimpleAlarmConditionExpression simpleExpression = new SimpleAlarmConditionExpression(); + AlarmConditionFilter filter = new AlarmConditionFilter(); + filter.setArgument("temperature"); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setOperation(NumericOperation.GREATER_OR_EQUAL); + AlarmConditionValue thresholdValue = new AlarmConditionValue<>(); + thresholdValue.setStaticValue(100.0); + predicate.setValue(thresholdValue); + filter.setPredicate(predicate); + simpleExpression.setFilters(List.of(filter)); + simpleExpression.setOperation(ComplexOperation.AND); + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition(simpleExpression, null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + + postTelemetry(deviceId, "{\"temperature\":100}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + /* * todo: state restore (event count) * */ @@ -310,21 +352,27 @@ public class AlarmRulesTest extends AbstractControllerTest { private AlarmRule toAlarmRule(Condition condition) { AlarmRule rule = new AlarmRule(); - TbelAlarmConditionExpression expression = new TbelAlarmConditionExpression(); - expression.setExpression(condition.expression()); - if (condition.eventsCount() != null) { + AlarmConditionExpression expression; + if (condition.getTbelExpression() != null) { + TbelAlarmConditionExpression tbelExpression = new TbelAlarmConditionExpression(); + tbelExpression.setExpression(condition.getTbelExpression()); + expression = tbelExpression; + } else { + expression = condition.getSimpleExpression(); + } + if (condition.getEventsCount() != null) { RepeatingAlarmCondition alarmCondition = new RepeatingAlarmCondition(); alarmCondition.setExpression(expression); AlarmConditionValue count = new AlarmConditionValue<>(); - count.setStaticValue(condition.eventsCount()); + count.setStaticValue(condition.getEventsCount()); alarmCondition.setCount(count); rule.setCondition(alarmCondition); - } else if (condition.durationMs() != null) { + } else if (condition.getDurationMs() != null) { DurationAlarmCondition alarmCondition = new DurationAlarmCondition(); alarmCondition.setExpression(expression); alarmCondition.setUnit(TimeUnit.MILLISECONDS); AlarmConditionValue duration = new AlarmConditionValue<>(); - duration.setStaticValue(condition.durationMs()); + duration.setStaticValue(condition.getDurationMs()); alarmCondition.setValue(duration); rule.setCondition(alarmCondition); } else { @@ -340,6 +388,28 @@ public class AlarmRulesTest extends AbstractControllerTest { .map(e -> (CalculatedFieldDebugEvent) e).toList(); } - private record Condition(String expression, Integer eventsCount, Long durationMs) {} + @Getter + private static final class Condition { + + private final String tbelExpression; + private final SimpleAlarmConditionExpression simpleExpression; + private final Integer eventsCount; + private final Long durationMs; + + private Condition(String tbelExpression, Integer eventsCount, Long durationMs) { + this.tbelExpression = tbelExpression; + this.simpleExpression = null; + this.eventsCount = eventsCount; + this.durationMs = durationMs; + } + + private Condition(SimpleAlarmConditionExpression simpleExpression, Integer eventsCount, Long durationMs) { + this.tbelExpression = null; + this.simpleExpression = simpleExpression; + this.eventsCount = eventsCount; + this.durationMs = durationMs; + } + + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java new file mode 100644 index 0000000000..aa70feca13 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.KeyFilterPredicate; + +import java.io.Serializable; + +@Schema +@Data +public class AlarmConditionFilter implements Serializable { + + @NotBlank + private String argument; + @Valid + @NotNull + private KeyFilterPredicate predicate; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/ComplexOperation.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/ComplexOperation.java new file mode 100644 index 0000000000..21c28fa552 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/ComplexOperation.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression; + +public enum ComplexOperation { + AND, + OR +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java index fe108c39e1..e541fbfd31 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java @@ -15,14 +15,19 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition.expression; -import jakarta.validation.constraints.NotBlank; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import lombok.Data; +import java.util.List; + @Data public class SimpleAlarmConditionExpression implements AlarmConditionExpression { - @NotBlank - private String expression; + @Valid + @NotEmpty + private List filters; + private ComplexOperation operation; @Override public AlarmConditionExpressionType getType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java new file mode 100644 index 0000000000..8a57aba3e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; + +@Data +public class BooleanFilterPredicate implements SimpleKeyFilterPredicate { + + private BooleanOperation operation; + private AlarmConditionValue value; + + @Override + public FilterPredicateType getType() { + return FilterPredicateType.BOOLEAN; + } + + public enum BooleanOperation { + EQUAL, + NOT_EQUAL + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/ComplexFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/ComplexFilterPredicate.java new file mode 100644 index 0000000000..4e24ea28ba --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/ComplexFilterPredicate.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.ComplexOperation; + +import java.util.List; + +@Data +public class ComplexFilterPredicate implements KeyFilterPredicate { + + private ComplexOperation operation; + private List predicates; + + @Override + public FilterPredicateType getType() { + return FilterPredicateType.COMPLEX; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/FilterPredicateType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/FilterPredicateType.java new file mode 100644 index 0000000000..af7c45ac5b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/FilterPredicateType.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +public enum FilterPredicateType { + STRING, + NUMERIC, + BOOLEAN, + COMPLEX +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/KeyFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/KeyFilterPredicate.java new file mode 100644 index 0000000000..58355c627d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/KeyFilterPredicate.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(value = StringFilterPredicate.class, name = "STRING"), + @Type(value = NumericFilterPredicate.class, name = "NUMERIC"), + @Type(value = BooleanFilterPredicate.class, name = "BOOLEAN"), + @Type(value = ComplexFilterPredicate.class, name = "COMPLEX")}) +public interface KeyFilterPredicate extends Serializable { + + @JsonIgnore + FilterPredicateType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java new file mode 100644 index 0000000000..30a82e06bb --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; + +@Data +public class NumericFilterPredicate implements SimpleKeyFilterPredicate { + + private NumericOperation operation; + private AlarmConditionValue value; + + @Override + public FilterPredicateType getType() { + return FilterPredicateType.NUMERIC; + } + + public enum NumericOperation { + EQUAL, + NOT_EQUAL, + GREATER, + LESS, + GREATER_OR_EQUAL, + LESS_OR_EQUAL + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/SimpleKeyFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/SimpleKeyFilterPredicate.java new file mode 100644 index 0000000000..0ea4cbf1eb --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/SimpleKeyFilterPredicate.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; + +public interface SimpleKeyFilterPredicate extends KeyFilterPredicate { + + AlarmConditionValue getValue(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java new file mode 100644 index 0000000000..ccc263611f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; + +@Data +public class StringFilterPredicate implements SimpleKeyFilterPredicate { + + private StringOperation operation; + private AlarmConditionValue value; + private boolean ignoreCase; + + @Override + public FilterPredicateType getType() { + return FilterPredicateType.STRING; + } + + public enum StringOperation { + EQUAL, + NOT_EQUAL, + STARTS_WITH, + ENDS_WITH, + CONTAINS, + NOT_CONTAINS, + IN, + NOT_IN + } +} From 0bdb3c0ef783b5ef940e18293e2cbb0ff08ca720 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 3 Oct 2025 13:25:11 +0300 Subject: [PATCH 010/122] Fix merge issues --- .../calculatedField/CalculatedFieldEntityMessageProcessor.java | 2 +- .../service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 2d3b1f5d40..1625becc20 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -488,7 +488,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (argNames.isEmpty()) { return Collections.emptyMap(); } - List geofencingArgumentNames = ctx.getLinkedEntityGeofencingArgumentNames(); + List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), geofencingArgumentNames, scope, removedAttrKeys); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 9b3f28cf3a..c071d8bf75 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -472,7 +472,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } @Override - protected void validateNewEntry(ArgumentEntry newEntry) { + protected void validateNewEntry(String key, ArgumentEntry newEntry) { if (!(newEntry instanceof SingleValueArgumentEntry)) { throw new IllegalArgumentException("Only single value arguments supported"); } From de988c977005c631a897a9011e0acbc38e082149 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 3 Oct 2025 14:55:37 +0300 Subject: [PATCH 011/122] Fix geofencing CF state init --- .../CalculatedFieldEntityMessageProcessor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1625becc20..b90b20fb4b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -352,6 +353,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void initState(CalculatedFieldState state, CalculatedFieldCtx ctx) { state.init(ctx); + if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.hasRelationQueryDynamicArguments()) { + GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; + geofencingState.setLastDynamicArgumentsRefreshTs(System.currentTimeMillis()); + } Map arguments = fetchArguments(ctx); state.update(arguments, ctx); From ce3b89046b6ec83e2e7291db053e3ed90d5cb8ed Mon Sep 17 00:00:00 2001 From: dshvaika Date: Fri, 3 Oct 2025 15:40:15 +0300 Subject: [PATCH 012/122] propagation cf init commit --- ...CalculatedFieldEntityMessageProcessor.java | 14 +++- ...tractCalculatedFieldProcessingService.java | 34 ++++++--- ...faultCalculatedFieldProcessingService.java | 41 ++++++++--- .../cf/PropagationCalculatedFieldResult.java | 49 +++++++++++++ .../service/cf/ctx/state/ArgumentEntry.java | 8 ++- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 39 ++++++---- .../geofencing/GeofencingArgumentEntry.java | 4 +- .../GeofencingCalculatedFieldState.java | 4 ++ .../propagation/PropagationArgumentEntry.java | 72 +++++++++++++++++++ .../PropagationCalculatedFieldState.java | 59 +++++++++++++++ .../utils/CalculatedFieldArgumentUtils.java | 2 + .../server/utils/CalculatedFieldUtils.java | 25 +++++-- .../common/data/cf/CalculatedFieldType.java | 3 +- .../BaseCalculatedFieldConfiguration.java | 10 ++- .../CalculatedFieldConfiguration.java | 3 +- ...opagationCalculatedFieldConfiguration.java | 66 +++++++++++++++++ common/proto/src/main/proto/queue.proto | 1 + .../script/api/tbel/TbelCfArg.java | 3 +- ...ncingArg.java => TbelCfGeofencingArg.java} | 4 +- .../script/api/tbel/TbelCfPropagationArg.java | 42 +++++++++++ 21 files changed, 434 insertions(+), 51 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/PropagationCalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfTsGeofencingArg.java => TbelCfGeofencingArg.java} (89%) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfPropagationArg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1625becc20..78ad6678a3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -319,13 +320,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state == null) { state = createState(ctx); justRestored = true; - } else if (ctx.shouldFetchDynamicArgumentsFromDb(state)) { + } else if (ctx.shouldFetchRelationQueryDynamicArgumentsFromDb(state)) { log.debug("[{}][{}] Going to update dynamic arguments for CF.", entityId, ctx.getCfId()); try { Map dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId); dynamicArgsFromDb.forEach(newArgValues::putIfAbsent); - var geofencingState = (GeofencingCalculatedFieldState) state; - geofencingState.setLastDynamicArgumentsRefreshTs(System.currentTimeMillis()); + if (ctx.getCfType() == CalculatedFieldType.GEOFENCING) { + var geofencingState = (GeofencingCalculatedFieldState) state; + geofencingState.updateLastDynamicArgumentsRefreshTs(); + } } catch (Exception e) { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } @@ -353,6 +356,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void initState(CalculatedFieldState state, CalculatedFieldCtx ctx) { state.init(ctx); + if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isRelationQueryDynamicArguments()) { + GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; + geofencingState.updateLastDynamicArgumentsRefreshTs(); + } + Map arguments = fetchArguments(ctx); state.update(arguments, ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 4018916582..232476c15a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -52,6 +52,8 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.cf.CalculatedFieldType.PROPAGATION; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; @@ -88,21 +90,26 @@ public abstract class AbstractCalculatedFieldProcessingService { protected ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { Map> argFutures = switch (ctx.getCalculatedField().getType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); - case SIMPLE, SCRIPT, ALARM -> { - Map> futures = new HashMap<>(); - for (var entry : ctx.getArguments().entrySet()) { - var argEntityId = resolveEntityId(ctx.getTenantId(), entityId, entry.getValue()); - var argValueFuture = fetchArgumentValue(ctx.getTenantId(), argEntityId, entry.getValue(), ts); - futures.put(entry.getKey(), argValueFuture); - } - yield futures; - } + case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); }; + if (ctx.getCfType() == PROPAGATION) { + argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); + } return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), MoreExecutors.directExecutor()); } + private Map> getBaseCalculatedFieldArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + Map> futures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = resolveEntityId(ctx.getTenantId(), entityId, entry.getValue()); + var argValueFuture = fetchArgumentValue(ctx.getTenantId(), argEntityId, entry.getValue(), ts); + futures.put(entry.getKey(), argValueFuture); + } + return futures; + } + protected EntityId resolveEntityId(TenantId tenantId, EntityId entityId, Argument argument) { if (argument.getRefEntityId() != null) { return argument.getRefEntityId(); @@ -130,6 +137,11 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } + protected ListenableFuture fetchPropagationCalculatedFieldArgument(CalculatedFieldCtx ctx, EntityId entityId) { + ListenableFuture> propagationEntityIds = fromDynamicSource(ctx.getTenantId(), entityId, ctx.getPropagationArgument()); + return Futures.transform(propagationEntityIds, ArgumentEntry::createPropagationArgument, MoreExecutors.directExecutor()); + } + protected Map> fetchGeofencingCalculatedFieldArguments(CalculatedFieldCtx ctx, EntityId entityId, boolean dynamicArgumentsOnly, long startTs) { Map> argFutures = new HashMap<>(); Set> entries = ctx.getArguments().entrySet(); @@ -160,6 +172,10 @@ public abstract class AbstractCalculatedFieldProcessingService { if (!value.hasDynamicSource()) { return Futures.immediateFuture(List.of(entityId)); } + return fromDynamicSource(tenantId, entityId, value); + } + + private ListenableFuture> fromDynamicSource(TenantId tenantId, EntityId entityId, Argument value) { var refDynamicSourceConfiguration = value.getRefDynamicSourceConfiguration(); return switch (refDynamicSourceConfiguration.getType()) { case CURRENT_OWNER -> Futures.immediateFuture(List.of(resolveOwnerArgument(tenantId, entityId))); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9b2964a736..52393d0ffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -23,7 +23,6 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -50,11 +49,13 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @TbRuleEngineComponent @@ -89,11 +90,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { - // only scheduledSupported CF instances supports dynamic arguments scheduled updates - if (!ctx.getCalculatedField().getType().equals(CalculatedFieldType.GEOFENCING)) { - return Map.of(); - } - return resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); + return switch (ctx.getCfType()) { + case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); + case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); + default -> Collections.emptyMap(); + }; } @Override @@ -112,13 +113,35 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback) { - try { + if (!(result instanceof PropagationCalculatedFieldResult propagationCalculatedFieldResult)) { TbMsg msg = result.toTbMsg(entityId, cfIds); + sendMsgToRuleEngine(tenantId, entityId, callback, msg); + return; + } + List propagationEntityIds = propagationCalculatedFieldResult.getPropagationEntityIds(); + if (propagationEntityIds.isEmpty()) { + callback.onSuccess(); + } + if (propagationEntityIds.size() == 1) { + EntityId propagationEntityId = propagationEntityIds.get(0); + TbMsg msg = result.toTbMsg(propagationEntityId, cfIds); + sendMsgToRuleEngine(tenantId, propagationEntityId, callback, msg); + return; + } + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback); + for (var propagationEntityId : propagationEntityIds) { + TbMsg msg = result.toTbMsg(propagationEntityId, cfIds); + sendMsgToRuleEngine(tenantId, propagationEntityId, multipleTbCallback, msg); + } + } + + private void sendMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbCallback callback, TbMsg msg) { + try { clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - callback.onSuccess(); log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + callback.onSuccess(); } @Override @@ -127,7 +150,7 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } }); } catch (Exception e) { - log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, result, e); + log.warn("[{}][{}] Failed to push message to rule engine: {}", tenantId, entityId, msg, e); callback.onFailure(e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/PropagationCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/PropagationCalculatedFieldResult.java new file mode 100644 index 0000000000..780fd220a7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/PropagationCalculatedFieldResult.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.List; + +@Data +@Builder +public final class PropagationCalculatedFieldResult implements CalculatedFieldResult { + + private final List propagationEntityIds; + private final TelemetryCalculatedFieldResult result; + + @Override + public TbMsg toTbMsg(EntityId entityId, List cfIds) { + return result.toTbMsg(entityId, cfIds); + } + + @Override + public String stringValue() { + return result.stringValue(); + } + + @Override + public boolean isEmpty() { + return CollectionsUtil.isEmpty(propagationEntityIds) || result.isEmpty(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 2d43883131..5f8276bc99 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; import java.util.List; import java.util.Map; @@ -35,7 +36,8 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING") + @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), + @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION") }) public interface ArgumentEntry { @@ -66,4 +68,8 @@ public interface ArgumentEntry { return new GeofencingArgumentEntry(entityIdkvEntryMap); } + static ArgumentEntry createPropagationArgument(List entityIds) { + return new PropagationArgumentEntry(entityIds); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 876bfa2a3f..2b118c9c07 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 12e4dc0a3a..ece459350f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ExpressionBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; @@ -101,6 +102,8 @@ public class CalculatedFieldCtx { private long scheduledUpdateIntervalMillis; + private Argument propagationArgument; + public CalculatedFieldCtx(CalculatedField calculatedField, ActorSystemContext systemContext) { this.calculatedField = calculatedField; @@ -154,6 +157,10 @@ public class CalculatedFieldCtx { } }); } + if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) { + propagationArgument = propagationConfig.toPropagationArgument(); + relationQueryDynamicArguments = true; + } } if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; @@ -170,7 +177,7 @@ public class CalculatedFieldCtx { public void init() { switch (cfType) { - case SCRIPT -> { + case SCRIPT, PROPAGATION -> { try { initTbelExpression(expression); initialized = true; @@ -512,21 +519,29 @@ public class CalculatedFieldCtx { return false; } - public boolean hasRelationQueryDynamicArguments() { - return relationQueryDynamicArguments && scheduledUpdateIntervalMillis != -1; + private boolean isScheduledUpdateEnabled() { + return scheduledUpdateIntervalMillis != -1; } - public boolean shouldFetchDynamicArgumentsFromDb(CalculatedFieldState state) { - if (!hasRelationQueryDynamicArguments()) { + public boolean shouldFetchRelationQueryDynamicArgumentsFromDb(CalculatedFieldState state) { + if (!relationQueryDynamicArguments) { return false; } - if (!(state instanceof GeofencingCalculatedFieldState geofencingState)) { - return false; - } - if (geofencingState.getLastDynamicArgumentsRefreshTs() == -1L) { - return true; - } - return geofencingState.getLastDynamicArgumentsRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis; + return switch (cfType) { + case PROPAGATION -> true; + case GEOFENCING -> { + if (!isScheduledUpdateEnabled()) { + yield false; + } + var geofencingState = (GeofencingCalculatedFieldState) state; + if (geofencingState.getLastDynamicArgumentsRefreshTs() == -1L) { + yield true; + } + yield geofencingState.getLastDynamicArgumentsRefreshTs() < + System.currentTimeMillis() - scheduledUpdateIntervalMillis; + } + default -> false; + }; } public void stop() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingArgumentEntry.java index 53e5c19e72..bcc4d3ffcd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingArgumentEntry.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state.geofencing; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfTsGeofencingArg; +import org.thingsboard.script.api.tbel.TbelCfGeofencingArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.util.ProtoUtils; @@ -83,7 +83,7 @@ public class GeofencingArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return new TbelCfTsGeofencingArg(zoneStates); + return new TbelCfGeofencingArg(zoneStates); } private Map toZones(Map entityIdKvEntryMap) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java index 47dce596da..f3bf8750cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java @@ -146,6 +146,10 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { lastDynamicArgumentsRefreshTs = -1; } + public void updateLastDynamicArgumentsRefreshTs() { + lastDynamicArgumentsRefreshTs = System.currentTimeMillis(); + } + private Map getGeofencingArguments() { return arguments.entrySet() .stream() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java new file mode 100644 index 0000000000..c7d49a4d40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.propagation; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfPropagationArg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; + +import java.util.List; + +@Data +public class PropagationArgumentEntry implements ArgumentEntry { + + private List propagationEntityIds; + + private boolean forceResetPrevious; + + public PropagationArgumentEntry(List propagationEntityIds) { + this.propagationEntityIds = propagationEntityIds; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.PROPAGATION; + } + + @Override + public Object getValue() { + return propagationEntityIds; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (!(entry instanceof PropagationArgumentEntry propagationArgumentEntry)) { + throw new IllegalArgumentException("Unsupported argument entry type for propagation argument entry: " + entry.getType()); + } + if (propagationArgumentEntry.isEmpty()) { + propagationEntityIds.clear(); + } else { + propagationEntityIds = propagationArgumentEntry.getPropagationEntityIds(); + } + return true; + } + + @Override + public boolean isEmpty() { + return CollectionsUtil.isEmpty(propagationEntityIds); + } + + @Override + public TbelCfArg toTbelCfArg() { + return new TbelCfPropagationArg(propagationEntityIds); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java new file mode 100644 index 0000000000..21a4493c91 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.propagation; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; + +import java.util.Map; + +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; + +public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState { + + public PropagationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.PROPAGATION; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { + ArgumentEntry argumentEntry = arguments.get(PROPAGATION_CONFIG_ARGUMENT); + if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry) || propagationArgumentEntry.isEmpty()) { + return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build()); + } + return Futures.transform(super.performCalculation(updatedArgs, ctx), telemetryCfResult -> + PropagationCalculatedFieldResult.builder() + .propagationEntityIds(propagationArgumentEntry.getPropagationEntityIds()) + .result((TelemetryCalculatedFieldResult) telemetryCfResult) + .build(), + MoreExecutors.directExecutor()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index c81d14f07b..df82488268 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -36,6 +36,7 @@ import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; import java.util.Optional; @@ -79,6 +80,7 @@ public class CalculatedFieldArgumentUtils { case SCRIPT -> new ScriptCalculatedFieldState(entityId); case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); + case PROPAGATION -> new PropagationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 38aeb45a20..69337fe2e6 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -32,6 +32,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.EntityIdProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; @@ -50,7 +51,10 @@ import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -58,6 +62,8 @@ import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; + public class CalculatedFieldUtils { public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { @@ -92,12 +98,11 @@ public class CalculatedFieldUtils { .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { - builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); - } else if (argEntry instanceof GeofencingArgumentEntry geofencingArgumentEntry) { - builder.addGeofencingArguments(toGeofencingArgumentProto(argName, geofencingArgumentEntry)); + switch (argEntry.getType()) { + case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case PROPAGATION -> builder.addAllPropagationEntityIds(toPropagationEntityIdsProto((PropagationArgumentEntry) argEntry)); } }); if (state instanceof AlarmCalculatedFieldState alarmState) { @@ -112,6 +117,10 @@ public class CalculatedFieldUtils { return builder.build(); } + private static List toPropagationEntityIdsProto(PropagationArgumentEntry argEntry) { + return argEntry.getPropagationEntityIds().stream().map(ProtoUtils::toProto).collect(Collectors.toList()); + } + private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) { return AlarmRuleStateProto.newBuilder() .setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse("")) @@ -178,11 +187,15 @@ public class CalculatedFieldUtils { case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); + case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + List propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList(); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds)); + switch (type) { case SCRIPT -> { proto.getRollingValueArgumentsList().forEach(argProto -> diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 3399808a35..7f38773c1e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -19,5 +19,6 @@ public enum CalculatedFieldType { SIMPLE, SCRIPT, GEOFENCING, - ALARM + ALARM, + PROPAGATION } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index b72cdad60a..6913b1ed63 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -28,12 +28,16 @@ public abstract class BaseCalculatedFieldConfiguration implements ExpressionBase @Override public void validate() { + baseCalculatedFieldRestriction(); + if (arguments.values().stream().anyMatch(Argument::hasRelationQuerySource)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support relation query configuration!"); + } + } + + protected void baseCalculatedFieldRestriction() { if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); } - if (arguments.values().stream().anyMatch(Argument::hasRelationQuerySource)) { - throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support relation query source configuration!"); - } } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..8676c6060f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -40,7 +40,8 @@ import java.util.stream.Collectors; @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), + @Type(value = PropagationCalculatedFieldConfiguration.class, name = "PROPAGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..7585c30438 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class PropagationCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration { + + public static final String PROPAGATION_CONFIG_ARGUMENT = "propagationCtx"; + + private EntitySearchDirection direction; + private String relationType; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.PROPAGATION; + } + + @Override + public void validate() { + baseCalculatedFieldRestriction(); + propagationRestriction(); + if (direction == null) { + throw new IllegalArgumentException("Propagation calculated field direction must be specified!"); + } + if (StringUtils.isBlank(relationType)) { + throw new IllegalArgumentException("Propagation calculated field relation type must be specified!"); + } + } + + public Argument toPropagationArgument() { + var refDynamicSourceConfiguration = new RelationPathQueryDynamicSourceConfiguration(); + refDynamicSourceConfiguration.setLevels(List.of(new RelationPathLevel(direction, relationType))); + var propagationArgument = new Argument(); + propagationArgument.setRefDynamicSourceConfiguration(refDynamicSourceConfiguration); + return propagationArgument; + } + + private void propagationRestriction() { + if (arguments.entrySet().stream().anyMatch(entry -> entry.getKey().equals(PROPAGATION_CONFIG_ARGUMENT))) { + throw new IllegalArgumentException("Argument name '" + PROPAGATION_CONFIG_ARGUMENT + "' is reserved and cannot be used."); + } + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index fac1116a30..1e3e121202 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -922,6 +922,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; + repeated EntityIdProto propagationEntityIds = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 73a2183564..6f83aac1b9 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -27,7 +27,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes({ @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = TbelCfTsGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsGeofencingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfGeofencingArg.java similarity index 89% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsGeofencingArg.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfGeofencingArg.java index f1e8ec16db..0fa0f4a5bf 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsGeofencingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfGeofencingArg.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data -public class TbelCfTsGeofencingArg implements TbelCfArg { +public class TbelCfGeofencingArg implements TbelCfArg { private final Object value; @JsonCreator - public TbelCfTsGeofencingArg(@JsonProperty("value") Object value) { + public TbelCfGeofencingArg(@JsonProperty("value") Object value) { this.value = value; } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfPropagationArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfPropagationArg.java new file mode 100644 index 0000000000..83d7e81a86 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfPropagationArg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfPropagationArg implements TbelCfArg { + + private final Object value; + + @JsonCreator + public TbelCfPropagationArg(@JsonProperty("value") Object value) { + this.value = value; + } + + @Override + public String getType() { + return "PROPAGATION_CF_ARGUMENT_VALUE"; + } + + @Override + public long memorySize() { + return OBJ_SIZE; + } + +} From 3a8bcc4f954fa56218cc1e2a002819e03acea763 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Mon, 6 Oct 2025 10:32:15 +0300 Subject: [PATCH 013/122] Fixed fromProto parsing in the CalculatedFieldUtils --- .../org/thingsboard/server/utils/CalculatedFieldUtils.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 69337fe2e6..51ca44fd54 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -193,9 +193,6 @@ public class CalculatedFieldUtils { proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - List propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList(); - state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds)); - switch (type) { case SCRIPT -> { proto.getRollingValueArgumentsList().forEach(argProto -> @@ -217,6 +214,10 @@ public class CalculatedFieldUtils { alarmState.getCreateRuleStates().put(severity, ruleState); } } + case PROPAGATION -> { + List propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList(); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds)); + } } return state; From 0b305e6f2a37b2e1f280ab95aaa6837196a7d3e4 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 11:16:46 +0300 Subject: [PATCH 014/122] Alarm rules CF: real-time duration condition checks --- .../server/actors/ActorSystemContext.java | 9 +- .../server/actors/app/AppActor.java | 1 + .../CalculatedFieldEntityActor.java | 4 + ...CalculatedFieldEntityMessageProcessor.java | 43 ++++-- .../CalculatedFieldManagerActor.java | 4 + ...alculatedFieldManagerMessageProcessor.java | 30 +--- .../CalculatedFieldReevaluateMsg.java | 2 +- .../CalculatedFieldStateRestoreMsg.java | 4 + .../server/actors/tenant/TenantActor.java | 2 + .../AbstractCalculatedFieldStateService.java | 35 ++++- .../ctx/state/BaseCalculatedFieldState.java | 18 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 15 +- .../cf/ctx/state/CalculatedFieldState.java | 12 +- .../KafkaCalculatedFieldStateService.java | 10 +- .../RocksDBCalculatedFieldStateService.java | 5 +- .../alarm/AlarmCalculatedFieldState.java | 133 ++++++++++++------ .../cf/ctx/state/alarm/AlarmEvalResult.java | 27 +++- .../cf/ctx/state/alarm/AlarmRuleState.java | 63 ++++++--- .../GeofencingCalculatedFieldState.java | 4 +- .../TbRuleEngineQueueConsumerManager.java | 3 +- .../server/utils/CalculatedFieldUtils.java | 24 ++-- .../src/main/resources/thingsboard.yml | 3 - .../thingsboard/server/cf/AlarmRulesTest.java | 7 +- .../GeofencingCalculatedFieldStateTest.java | 3 +- .../state/ScriptCalculatedFieldStateTest.java | 3 +- .../state/SimpleCalculatedFieldStateTest.java | 3 +- .../TbRuleEngineQueueConsumerManagerTest.java | 19 ++- .../ruleengine/TbRuleEngineStrategyTest.java | 4 +- .../server/queue/TbQueueConsumer.java | 2 + .../AlarmCalculatedFieldConfiguration.java | 7 - .../CalculatedFieldConfiguration.java | 4 - ...lculatedFieldStatePartitionRestoreMsg.java | 37 +++++ .../server/common/msg/MsgType.java | 1 + common/proto/src/main/proto/queue.proto | 6 +- .../AbstractTbQueueConsumerTemplate.java | 5 + .../consumer/MainQueueConsumerManager.java | 17 ++- .../common/consumer/TbQueueConsumerTask.java | 18 ++- .../state/DefaultQueueStateService.java | 19 +++ .../common/state/KafkaQueueStateService.java | 9 +- .../queue/common/state/QueueStateService.java | 26 ++-- .../queue/memory/InMemoryTbQueueConsumer.java | 5 + 41 files changed, 449 insertions(+), 197 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/CalculatedFieldStatePartitionRestoreMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 38c409036d..f3e636296a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -664,10 +664,6 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; - @Value("${actors.alarms.reevaluation_interval:60}") - @Getter - private long alarmsReevaluationInterval; - @Autowired @Getter private MqttClientSettings mqttClientSettings; @@ -895,12 +891,13 @@ public class ActorSystemContext { return getScheduler().scheduleWithFixedDelay(() -> ctx.tell(msg), delayInMs, periodInMs, TimeUnit.MILLISECONDS); } - public void scheduleMsgWithDelay(TbActorRef ctx, TbActorMsg msg, long delayInMs) { + public ScheduledFuture scheduleMsgWithDelay(TbActorRef ctx, TbActorMsg msg, long delayInMs) { log.debug("Scheduling msg {} with delay {} ms", msg, delayInMs); if (delayInMs > 0) { - getScheduler().schedule(() -> ctx.tell(msg), delayInMs, TimeUnit.MILLISECONDS); + return getScheduler().schedule(() -> ctx.tell(msg), delayInMs, TimeUnit.MILLISECONDS); } else { ctx.tell(msg); + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index e515d58695..4715ea64d4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -88,6 +88,7 @@ public class AppActor extends ContextAwareActor { break; case PARTITION_CHANGE_MSG: case CF_PARTITIONS_CHANGE_MSG: + case CF_STATE_PARTITION_RESTORE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index e2a2b93436..cababd4b6d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -21,6 +21,7 @@ import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @@ -63,6 +64,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; + case CF_STATE_PARTITION_RESTORE_MSG: + processor.process((CalculatedFieldStatePartitionRestoreMsg) msg); + break; case CF_ENTITY_INIT_CF_MSG: processor.process((EntityInitCalculatedFieldMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index b90b20fb4b..0a175b4899 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -83,7 +84,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final CalculatedFieldProcessingService cfService; final CalculatedFieldStateService cfStateService; - TbActorCtx ctx; + TbActorCtx actorCtx; Map states = new HashMap<>(); CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { @@ -95,7 +96,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } void init(TbActorCtx ctx) { - this.ctx = ctx; + this.actorCtx = ctx; } public void stop(boolean partitionChanged) { @@ -104,7 +105,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM "[{}][{}] Stopping entity actor.", tenantId, entityId); states.clear(); - ctx.stop(ctx.getSelf()); + actorCtx.stop(actorCtx.getSelf()); } public void process(CalculatedFieldPartitionChangeMsg msg) { @@ -116,13 +117,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(CalculatedFieldStateRestoreMsg msg) { CalculatedFieldId cfId = msg.getId().cfId(); log.debug("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); - if (msg.getState() != null) { - states.put(cfId, msg.getState()); + CalculatedFieldState state = msg.getState(); + if (state != null) { + state.setCtx(msg.getCtx(), actorCtx); + state.setPartition(msg.getPartition()); + states.put(cfId, state); } else { states.remove(cfId); } } + public void process(CalculatedFieldStatePartitionRestoreMsg msg) { + log.debug("Processing CF state partition restore msg: {}", msg); + for (CalculatedFieldState state : states.values()) { + if (msg.getPartition().equals(state.getPartition())) { + state.init(); + } + } + } + public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing entity init CF msg: {}", msg.getCtx().getCfId(), msg); var ctx = msg.getCtx(); @@ -138,10 +151,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = createState(ctx); } else if (msg.getStateAction() == StateAction.REINIT) { log.debug("Force reinitialization of CF: [{}].", ctx.getCfId()); - state.reset(ctx); + state.reset(); initState(state, ctx); } else { - state.init(ctx); + state.setCtx(ctx, actorCtx); + state.init(); } if (state.isSizeOk()) { processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); @@ -183,7 +197,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); - ctx.stop(ctx.getSelf()); + actorCtx.stop(actorCtx.getSelf()); } } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); @@ -266,30 +280,30 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldReevaluateMsg msg) throws CalculatedFieldException { - CalculatedFieldId cfId = msg.getCfCtx().getCfId(); + CalculatedFieldId cfId = msg.getCtx().getCfId(); CalculatedFieldState state = states.get(cfId); if (state == null) { log.debug("[{}][{}] Failed to find CF state for entity to handle {}", entityId, cfId, msg); } else { if (state.isSizeOk()) { log.debug("[{}][{}] Reevaluating CF state", entityId, cfId); - processStateIfReady(state, null, msg.getCfCtx(), Collections.singletonList(cfId), null, null, msg.getCallback()); + processStateIfReady(state, null, msg.getCtx(), Collections.singletonList(cfId), null, null, msg.getCallback()); } else { - throw new RuntimeException(msg.getCfCtx().getSizeExceedsLimitMessage()); + throw new RuntimeException(msg.getCtx().getSizeExceedsLimitMessage()); } } } public void process(CalculatedFieldAlarmActionMsg msg) { log.debug("[{}] Processing alarm action event msg: {}", entityId, msg); - states.values().forEach(state -> { + for (CalculatedFieldState state : states.values()) { if (state instanceof AlarmCalculatedFieldState alarmCfState) { Alarm stateAlarm = alarmCfState.getCurrentAlarm(); if (stateAlarm != null && stateAlarm.getId().equals(msg.getAlarm().getId())) { alarmCfState.processAlarmAction(msg.getAlarm(), msg.getAction()); } } - }); + } msg.getCallback().onSuccess(); } @@ -352,7 +366,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private void initState(CalculatedFieldState state, CalculatedFieldCtx ctx) { - state.init(ctx); + state.setCtx(ctx, actorCtx); + state.init(); if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.hasRelationQueryDynamicArguments()) { GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; geofencingState.setLastDynamicArgumentsRefreshTs(System.currentTimeMillis()); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index f1e15866eb..80daff07ef 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -20,6 +20,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; @@ -70,6 +71,9 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { case CF_STATE_RESTORE_MSG: processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); break; + case CF_STATE_PARTITION_RESTORE_MSG: + processor.onStatePartitionRestoreMsg((CalculatedFieldStatePartitionRestoreMsg) msg); + break; case CF_ENTITY_LIFECYCLE_MSG: processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d43a685c89..5e7437d12d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @@ -63,8 +64,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -79,7 +78,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); - private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; @@ -120,10 +118,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.clear(); entityIdCalculatedFields.clear(); entityIdCalculatedFieldLinks.clear(); - if (cfsReevaluationTask != null) { - cfsReevaluationTask.cancel(true); - cfsReevaluationTask = null; - } ctx.stop(ctx.getSelf()); } @@ -131,7 +125,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); initEntitiesCache(); initCalculatedFields(); - scheduleCfsReevaluation(); msg.getCallback().onSuccess(); } @@ -140,9 +133,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var ctx = calculatedFields.get(cfId); if (ctx != null) { - if (msg.getState() != null) { - msg.getState().init(ctx); - } + msg.setCtx(ctx); log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { @@ -150,21 +141,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } - private void scheduleCfsReevaluation() { - cfsReevaluationTask = systemContext.getScheduler().scheduleWithFixedDelay(() -> { - try { - calculatedFields.values().forEach(cf -> { - if (cf.isRequiresScheduledReevaluation()) { - applyToTargetCfEntityActors(cf, TbCallback.EMPTY, (entityId, callback) -> { - log.debug("[{}][{}] Pushing scheduled CF reevaluate msg", entityId, cf.getCfId()); - getOrCreateActor(entityId).tell(new CalculatedFieldReevaluateMsg(tenantId, cf)); - }); - } - }); - } catch (Exception e) { - log.warn("[{}] Failed to trigger CFs reevaluation", tenantId, e); - } - }, systemContext.getAlarmsReevaluationInterval(), systemContext.getAlarmsReevaluationInterval(), TimeUnit.SECONDS); + public void onStatePartitionRestoreMsg(CalculatedFieldStatePartitionRestoreMsg msg) { + ctx.broadcastToChildren(msg, true); } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java index a0b75d1a72..b617736ee0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldReevaluateMsg.java @@ -25,7 +25,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; public class CalculatedFieldReevaluateMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; - private final CalculatedFieldCtx cfCtx; + private final CalculatedFieldCtx ctx; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java index 19be7c02fa..d1c2f11aeb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -19,7 +19,9 @@ import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data @@ -27,6 +29,8 @@ public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMs private final CalculatedFieldEntityCtxId id; private final CalculatedFieldState state; + private final TopicPartitionInfo partition; + private CalculatedFieldCtx ctx; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 35cbf3ccd8..e33965a42e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -186,6 +186,7 @@ public class TenantActor extends RuleChainManagerActor { case CF_CACHE_INIT_MSG: case CF_STATE_RESTORE_MSG: case CF_PARTITIONS_CHANGE_MSG: + case CF_STATE_PARTITION_RESTORE_MSG: forwardToCfActor((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: @@ -394,6 +395,7 @@ public class TenantActor extends RuleChainManagerActor { public TbActor createActor() { return new TenantActor(context, tenantId); } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index a77eb71343..c8b99afd9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.service.cf; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.exception.TenantNotFoundException; +import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.exception.CalculatedFieldStateException; @@ -37,6 +42,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; +@Slf4j public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { @Autowired @@ -62,19 +68,38 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); - protected void processRestoredState(CalculatedFieldStateProto stateMsg) { + protected void processRestoredState(CalculatedFieldStateProto stateMsg, TopicPartitionInfo partition) { var id = fromProto(stateMsg.getId()); + if (partition == null) { + try { + partition = actorSystemContext.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, id.tenantId(), id.entityId()); + } catch (TenantNotFoundException e) { + log.debug("Skipping CF state msg for non-existing tenant {}", id.tenantId()); + return; + } + } var state = fromProto(id, stateMsg); - processRestoredState(id, state); + processRestoredState(id, state, partition); } - protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { - actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); + protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state, TopicPartitionInfo partition) { + partition = partition.withTopic(DataConstants.CF_STATES_QUEUE_NAME); + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state, partition)); } @Override public void restore(QueueKey queueKey, Set partitions) { - stateService.update(queueKey, partitions, null); + stateService.update(queueKey, partitions, new QueueStateService.RestoreCallback() { + @Override + public void onAllPartitionsRestored() { + } + + @Override + public void onPartitionRestored(TopicPartitionInfo partition) { + partition = partition.withTopic(DataConstants.CF_STATES_QUEUE_NAME); + actorSystemContext.tellWithHighPriority(new CalculatedFieldStatePartitionRestoreMsg(partition)); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d027a2f9fe..153b83f8d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -16,7 +16,10 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; @@ -29,21 +32,32 @@ import java.util.Map; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected final EntityId entityId; + protected CalculatedFieldCtx ctx; + protected TbActorRef actorCtx; protected List requiredArguments; protected Map arguments = new HashMap<>(); protected boolean sizeExceedsLimit; protected long latestTimestamp = -1; + @Setter + private TopicPartitionInfo partition; + public BaseCalculatedFieldState(EntityId entityId) { this.entityId = entityId; } @Override - public void init(CalculatedFieldCtx ctx) { + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + this.ctx = ctx; + this.actorCtx = actorCtx; this.requiredArguments = ctx.getArgNames(); } + @Override + public void init() { + } + @Override public Map update(Map argumentValues, CalculatedFieldCtx ctx) { Map updatedArguments = null; @@ -82,7 +96,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public void reset(CalculatedFieldCtx ctx) { // must reset everything dependent on arguments + public void reset() { // must reset everything dependent on arguments requiredArguments = null; arguments.clear(); sizeExceedsLimit = false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 12e4dc0a3a..db755d3e17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import net.objecthunter.exp4j.Expression; import org.mvel2.MVEL; import org.thingsboard.common.util.ExpressionUtils; @@ -25,6 +26,8 @@ import org.thingsboard.script.api.tbel.TbelCfCtx; import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldReevaluateMsg; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; @@ -61,9 +64,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.concurrent.ScheduledFuture; import java.util.stream.Stream; @Data +@Slf4j public class CalculatedFieldCtx { private CalculatedField calculatedField; @@ -80,8 +85,8 @@ public class CalculatedFieldCtx { private Output output; private String expression; private boolean useLatestTs; - private boolean requiresScheduledReevaluation; + private ActorSystemContext systemContext; private TbelInvokeService tbelInvokeService; private RelationService relationService; private AlarmSubscriptionService alarmService; @@ -158,7 +163,7 @@ public class CalculatedFieldCtx { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } - this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); this.alarmService = systemContext.getAlarmService(); @@ -236,6 +241,12 @@ public class CalculatedFieldCtx { return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); } + public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { + log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); + // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg + return systemContext.scheduleMsgWithDelay(actorCtx, new CalculatedFieldReevaluateMsg(tenantId, this), delayMs); + } + private TbelCfArg toTbelArgument(String key, CalculatedFieldState state) { return state.getArguments().get(key).toTbelCfArg(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index ad8005af83..ff94206220 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,7 +20,9 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; @@ -47,11 +49,13 @@ public interface CalculatedFieldState { long getLatestTimestamp(); - void init(CalculatedFieldCtx ctx); + void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); + + void init(); Map update(Map arguments, CalculatedFieldCtx ctx); - void reset(CalculatedFieldCtx ctx); + void reset(); ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); @@ -65,6 +69,10 @@ public interface CalculatedFieldState { return !isSizeExceedsLimit(); } + TopicPartitionInfo getPartition(); + + void setPartition(TopicPartitionInfo partition); + void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); default void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 2b52892744..e8174bfd57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -43,6 +43,7 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToString; @@ -77,9 +78,9 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta for (TbProtoQueueMsg msg : msgs) { try { if (msg.getValue() != null) { - processRestoredState(msg.getValue()); + processRestoredState(msg.getValue(), consumerKey.partition()); } else { - processRestoredState(getStateId(msg.getHeaders()), null); + processRestoredState(getStateId(msg.getHeaders()), null, consumerKey.partition()); } } catch (Throwable t) { log.error("Failed to process state message: {}", msg, t); @@ -104,6 +105,11 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } + @Override + public void restore(QueueKey queueKey, Set partitions) { + stateService.update(queueKey, partitions, null); + } + @Override protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME, stateId.tenantId(), stateId.entityId()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 9dc6139ca5..05bfb8b717 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -64,8 +63,8 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS if (stateService.getPartitions().isEmpty()) { cfRocksDb.forEach((key, value) -> { try { - processRestoredState(CalculatedFieldStateProto.parseFrom(value)); - } catch (InvalidProtocolBufferException e) { + processRestoredState(CalculatedFieldStateProto.parseFrom(value), null); + } catch (Exception e) { log.error("[{}] Failed to process restored state", key, e); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index c071d8bf75..838cb2779d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -21,11 +21,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; +import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -61,10 +63,15 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Comparator; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import static org.thingsboard.server.common.data.StringUtils.equalsAny; import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.service.cf.ctx.state.alarm.AlarmEvalResult.Status.FALSE; +import static org.thingsboard.server.service.cf.ctx.state.alarm.AlarmEvalResult.Status.NOT_YET_TRUE; +import static org.thingsboard.server.service.cf.ctx.state.alarm.AlarmEvalResult.Status.TRUE; @EqualsAndHashCode(callSuper = true) @Slf4j @@ -76,6 +83,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Getter private final Map createRuleStates = new TreeMap<>(Comparator.comparing(Enum::ordinal)); @Getter + @Setter private AlarmRuleState clearRuleState; @Getter @@ -87,36 +95,71 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public void init(CalculatedFieldCtx ctx) { - super.init(ctx); - + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); this.alarmType = ctx.getCalculatedField().getName(); this.configuration = getConfiguration(ctx); + } + @Override + public void init() { // todo: properly close state! + super.init(); + AtomicBoolean reevalNeeded = new AtomicBoolean(false); Map createRules = configuration.getCreateRules(); - createRules.forEach((severity, rule) -> { - AlarmRuleState ruleState = createRuleStates.get(severity); - if (ruleState == null) { - ruleState = new AlarmRuleState(severity, rule, this); - createRuleStates.put(severity, ruleState); - } else { // can be null if was restored - ruleState.setAlarmRule(rule); - // todo: is it enough to just set new alarm rule to alarm rule state? is it ok to leave the state as were?? + for (AlarmSeverity severity : AlarmSeverity.values()) { + AlarmRule rule = createRules.get(severity); + if (rule != null) { + createRuleStates.compute(severity, (__, ruleState) -> { + return initRuleState(severity, rule, ruleState, reevalNeeded); + }); + } else { + AlarmRuleState state = createRuleStates.remove(severity); + if (state != null) { + clearState(state); + } } - }); - createRuleStates.keySet().removeIf(severity -> !createRules.containsKey(severity)); + } AlarmRule clearRule = configuration.getClearRule(); if (clearRule != null) { - if (clearRuleState == null) { - clearRuleState = new AlarmRuleState(null, clearRule, this); - } else { - clearRuleState.setAlarmRule(clearRule); + clearRuleState = initRuleState(null, clearRule, clearRuleState, reevalNeeded); + } else { + if (clearRuleState != null) { + clearState(clearRuleState); + clearRuleState = null; } + } + log.debug("Initialized create rule states {} and clear rule state {} for {}", createRuleStates, clearRuleState, configuration); + + if (reevalNeeded.get()) { + initCurrentAlarm(ctx); + createOrClearAlarms(state -> { + if (state.getCondition().getType() == AlarmConditionType.DURATION) { + AlarmEvalResult evalResult = state.reeval(System.currentTimeMillis()); + if (evalResult.getStatus() == TRUE || evalResult.getStatus() == NOT_YET_TRUE) { + ScheduledFuture future = ctx.scheduleReevaluation(evalResult.getLeftDuration(), actorCtx); + // TODO: use single task for multiple durations if durations are close enough. but be careful when cancelling the task in one of the states + if (future != null) { + state.setDurationCheckFuture(future); + } + } + } + return AlarmEvalResult.NOT_YET_TRUE; + }, ctx); + } + } + + private AlarmRuleState initRuleState(AlarmSeverity severity, AlarmRule rule, AlarmRuleState ruleState, AtomicBoolean reevalNeeded) { + if (ruleState == null) { + ruleState = new AlarmRuleState(severity, rule, this); } else { - clearRuleState = null; + // when restored + ruleState.setAlarmRule(rule); + if (rule.getCondition().getType() == AlarmConditionType.DURATION && !ruleState.isEmpty()) { + reevalNeeded.set(true); + } } - log.debug("Initialized create rule states {} and clear rule state {} for {}", createRuleStates, clearRuleState, ctx.getCalculatedField()); + return ruleState; } @Override @@ -125,8 +168,12 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public void reset(CalculatedFieldCtx ctx) { - super.reset(ctx); + public void reset() { + super.reset(); + createRuleStates.values().forEach(AlarmRuleState::clear); + if (clearRuleState != null) { + clearRuleState.clear(); + } } @Override @@ -135,9 +182,19 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { TbAlarmResult result = createOrClearAlarms(state -> { if (updatedArgs != null) { boolean newEvent = !updatedArgs.isEmpty(); - return state.eval(newEvent, ctx); + AlarmEvalResult evalResult = state.eval(newEvent, ctx); + if (evalResult.getStatus() == NOT_YET_TRUE && evalResult.getLeftDuration() > 0) { + // rounding up to the closest second +// long leftDuration = (long) Math.ceil(evalResult.getLeftDuration() / 1000.0) * 1000; + long leftDuration = evalResult.getLeftDuration(); + ScheduledFuture future = ctx.scheduleReevaluation(leftDuration, actorCtx); // TODO: use single task for multiple durations if durations are close enough. but be careful when cancelling the task in one of the states + if (future != null) { + state.setDurationCheckFuture(future); + } + } + return evalResult; } else { - return state.eval(System.currentTimeMillis()); + return state.reeval(System.currentTimeMillis()); } }, ctx); return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() @@ -177,11 +234,11 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { for (AlarmRuleState state : createRuleStates.values()) { AlarmEvalResult evalResult = evalFunction.apply(state); log.debug("Evaluated create rule {} with args {}. Result: {}", state, arguments, evalResult); - if (evalResult == AlarmEvalResult.TRUE) { + if (evalResult.getStatus() == TRUE) { resultState = state; break; - } else if (evalResult == AlarmEvalResult.FALSE) { - clearAlarmState(state); + } else if (evalResult.getStatus() == FALSE) { + clearState(state); } } @@ -189,15 +246,15 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { result = calculateAlarmResult(resultState, ctx); resultStateInfo = resultState.getStateInfo(); log.debug("Alarm result for state {}: {}", resultState, result); - clearAlarmState(clearRuleState); + clearState(clearRuleState); } else if (currentAlarm != null && clearRuleState != null) { AlarmEvalResult evalResult = evalFunction.apply(clearRuleState); log.debug("Evaluated clear rule {} with args {}. Result: {}", clearRuleState, arguments, evalResult); - if (evalResult == AlarmEvalResult.TRUE) { + if (evalResult.getStatus() == TRUE) { resultStateInfo = clearRuleState.getStateInfo(); - clearAlarmState(clearRuleState); + clearState(clearRuleState); for (AlarmRuleState state : createRuleStates.values()) { - clearAlarmState(state); + clearState(state); } AlarmApiCallResult clearResult = ctx.getAlarmService().clearAlarm( ctx.getTenantId(), currentAlarm.getId(), System.currentTimeMillis(), createDetails(clearRuleState), true @@ -207,12 +264,11 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { .isCleared(true) .alarm(clearResult.getAlarm()) .build(); - addStateInfo(result, clearRuleState); resultState = clearRuleState; } currentAlarm = null; - } else if (evalResult == AlarmEvalResult.FALSE) { - clearAlarmState(clearRuleState); + } else if (evalResult.getStatus() == FALSE) { + clearState(clearRuleState); } } if (result != null && resultState != null) { @@ -222,8 +278,9 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { return result; } - private void clearAlarmState(AlarmRuleState state) { + private void clearState(AlarmRuleState state) { if (state != null) { + log.debug("Clearing rule state {}", state); state.clear(); } } @@ -283,14 +340,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } } - private void addStateInfo(TbAlarmResult alarmResult, AlarmRuleState ruleState) { - if (ruleState.getCondition().getType() == AlarmConditionType.REPEATING) { - alarmResult.setConditionRepeats(ruleState.getEventCount()); - } else if (ruleState.getCondition().getType() == AlarmConditionType.DURATION) { - alarmResult.setConditionDuration(ruleState.getDuration()); - } - } - private JsonNode createDetails(AlarmRuleState ruleState) { JsonNode alarmDetails; String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java index 6775b14586..424a977c75 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmEvalResult.java @@ -15,8 +15,31 @@ */ package org.thingsboard.server.service.cf.ctx.state.alarm; -public enum AlarmEvalResult { +import lombok.Data; +import lombok.RequiredArgsConstructor; - FALSE, NOT_YET_TRUE, TRUE; +@Data +@RequiredArgsConstructor +public class AlarmEvalResult { + + public static final AlarmEvalResult TRUE = new AlarmEvalResult(Status.TRUE); + public static final AlarmEvalResult FALSE = new AlarmEvalResult(Status.FALSE); + public static final AlarmEvalResult NOT_YET_TRUE = new AlarmEvalResult(Status.NOT_YET_TRUE); + + private final Status status; + private final long leftDuration; + private final long leftEvents; + + public AlarmEvalResult(Status status) { + this(status, 0, 0); + } + + public static AlarmEvalResult notYetTrue(long leftEvents, long leftDuration) { + return new AlarmEvalResult(Status.NOT_YET_TRUE, leftDuration, leftEvents); + } + + public enum Status { + FALSE, NOT_YET_TRUE, TRUE; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 0e5459af5e..0638543685 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -38,6 +38,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Optional; +import java.util.concurrent.ScheduledFuture; @Data @Slf4j @@ -49,9 +50,11 @@ public class AlarmRuleState { private AlarmCondition condition; - private long lastEventTs; - private long duration; private long eventCount; + private long firstEventTs; // when duration condition started + private long lastEventTs; + private transient long duration; + private ScheduledFuture durationCheckFuture; public AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, AlarmCalculatedFieldState state) { this.severity = severity; @@ -70,17 +73,22 @@ public class AlarmRuleState { }; } - public AlarmEvalResult eval(long ts) { // on schedule + public AlarmEvalResult reeval(long ts) { switch (condition.getType()) { case SIMPLE, REPEATING -> { return AlarmEvalResult.NOT_YET_TRUE; } case DURATION -> { - long requiredDurationInMs = getRequiredDurationInMs(); - if (requiredDurationInMs > 0 && lastEventTs > 0 && ts > lastEventTs) { - long duration = this.duration + (ts - lastEventTs); + long requiredDuration = getRequiredDurationInMs(); + if (requiredDuration > 0 && lastEventTs > 0 && ts > lastEventTs) { + duration = ts - firstEventTs; if (isActive(ts)) { - return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + long leftDuration = requiredDuration - duration; + if (leftDuration <= 0) { + return AlarmEvalResult.TRUE; + } else { + return AlarmEvalResult.notYetTrue(0, leftDuration); + } } else { return AlarmEvalResult.FALSE; } @@ -101,7 +109,8 @@ public class AlarmRuleState { eventCount++; } long requiredRepeats = getIntValue(((RepeatingAlarmCondition) condition).getCount()); - return eventCount >= requiredRepeats ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; + long leftRepeats = requiredRepeats - eventCount; + return leftRepeats <= 0 ? AlarmEvalResult.TRUE : AlarmEvalResult.notYetTrue(leftRepeats, 0); } else { return AlarmEvalResult.FALSE; } @@ -109,17 +118,26 @@ public class AlarmRuleState { private AlarmEvalResult evalDuration(boolean active, CalculatedFieldCtx ctx) { if (active && eval(condition.getExpression(), ctx)) { + long eventTs = state.getLatestTimestamp(); if (lastEventTs > 0) { - if (state.getLatestTimestamp() > lastEventTs) { - duration = duration + (state.getLatestTimestamp() - lastEventTs); - lastEventTs = state.getLatestTimestamp(); + if (eventTs > lastEventTs) { + if (firstEventTs == 0) { + firstEventTs = lastEventTs; + } + lastEventTs = eventTs; } } else { - lastEventTs = state.getLatestTimestamp(); - duration = 0L; + firstEventTs = eventTs; + lastEventTs = eventTs; + } + duration = lastEventTs - firstEventTs; + long requiredDuration = getRequiredDurationInMs(); + long leftDuration = requiredDuration - duration; + if (leftDuration <= 0) { + return AlarmEvalResult.TRUE; + } else { + return AlarmEvalResult.notYetTrue(0, leftDuration); } - long requiredDurationInMs = getRequiredDurationInMs(); - return duration > requiredDurationInMs ? AlarmEvalResult.TRUE : AlarmEvalResult.NOT_YET_TRUE; } else { return AlarmEvalResult.FALSE; } @@ -190,8 +208,17 @@ public class AlarmRuleState { public void clear() { eventCount = 0L; + firstEventTs = 0L; lastEventTs = 0L; duration = 0L; + if (durationCheckFuture != null) { + durationCheckFuture.cancel(true); + durationCheckFuture = null; + } + } + + public boolean isEmpty() { + return eventCount == 0L && firstEventTs == 0L && lastEventTs == 0L && durationCheckFuture == null; } private Integer getIntValue(AlarmConditionValue value) { @@ -216,7 +243,7 @@ public class AlarmRuleState { if (condition.getType() == AlarmConditionType.REPEATING) { return new StateInfo(eventCount, null); } else if (condition.getType() == AlarmConditionType.DURATION) { - return new StateInfo(null, duration + (System.currentTimeMillis() - lastEventTs)); + return new StateInfo(null, duration); } else { return StateInfo.EMPTY; } @@ -227,9 +254,11 @@ public class AlarmRuleState { return "AlarmRuleState{" + "severity=" + severity + ", condition=" + condition + + ", eventCount=" + eventCount + + ", firstEventTs=" + firstEventTs + ", lastEventTs=" + lastEventTs + ", duration=" + duration + - ", eventCount=" + eventCount + + ", durationCheckFuture=" + durationCheckFuture + '}'; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java index 47dce596da..6f4d40ca0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java @@ -141,8 +141,8 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public void reset(CalculatedFieldCtx ctx) { - super.reset(ctx); + public void reset() { + super.reset(); lastDynamicArgumentsRefreshTs = -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index d067be49a0..ff8d1cee2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -38,6 +38,7 @@ import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.DeleteQueueTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.ConsumerKey; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; @@ -127,7 +128,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager> msgs, TbQueueConsumer> consumer, - Object consumerKey, + ConsumerKey consumerKey, Queue queue) throws Exception { TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 38aeb45a20..4319f72448 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -115,12 +115,21 @@ public class CalculatedFieldUtils { private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) { return AlarmRuleStateProto.newBuilder() .setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse("")) - .setLastEventTs(ruleState.getLastEventTs()) - .setDuration(ruleState.getDuration()) .setEventCount(ruleState.getEventCount()) + .setFirstEventTs(ruleState.getFirstEventTs()) + .setLastEventTs(ruleState.getLastEventTs()) .build(); } + private static AlarmRuleState fromAlarmRuleStateProto(AlarmRuleStateProto proto, AlarmCalculatedFieldState state) { + AlarmSeverity severity = StringUtils.isNotEmpty(proto.getSeverity()) ? AlarmSeverity.valueOf(proto.getSeverity()) : null; + AlarmRuleState ruleState = new AlarmRuleState(severity, null, state); + ruleState.setEventCount(proto.getEventCount()); + ruleState.setFirstEventTs(proto.getFirstEventTs()); + ruleState.setLastEventTs(proto.getLastEventTs()); + return ruleState; + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -196,12 +205,11 @@ public class CalculatedFieldUtils { AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state; AlarmStateProto alarmStateProto = proto.getAlarmState(); for (AlarmRuleStateProto ruleStateProto : alarmStateProto.getCreateRuleStatesList()) { - AlarmSeverity severity = StringUtils.isNotEmpty(ruleStateProto.getSeverity()) ? AlarmSeverity.valueOf(ruleStateProto.getSeverity()) : null; - AlarmRuleState ruleState = new AlarmRuleState(severity, null, alarmState); - ruleState.setLastEventTs(ruleStateProto.getLastEventTs()); - ruleState.setDuration(ruleStateProto.getDuration()); - ruleState.setEventCount(ruleStateProto.getEventCount()); - alarmState.getCreateRuleStates().put(severity, ruleState); + AlarmRuleState ruleState = fromAlarmRuleStateProto(ruleStateProto, alarmState); + alarmState.getCreateRuleStates().put(ruleState.getSeverity(), ruleState); + } + if (alarmStateProto.hasClearRuleState()) { + alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 561275c2a6..192cb242d0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -526,9 +526,6 @@ actors: configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" - alarms: - # Interval in seconds to re-evaluate Alarm rules with duration condition - reevaluation_interval: "${ACTORS_ALARMS_REEVALUATION_INTERVAL_SEC:60}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 1a28a5aef8..b0b28f06dc 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -21,7 +21,6 @@ import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; @@ -72,9 +71,6 @@ import static org.testcontainers.shaded.org.awaitility.Awaitility.await; @Slf4j @DaoSqlTest -@TestPropertySource(properties = { - "actors.alarms.reevaluation_interval=1" -}) public class AlarmRulesTest extends AbstractControllerTest { @MockitoSpyBean @@ -235,10 +231,9 @@ public class AlarmRulesTest extends AbstractControllerTest { Map createRules = Map.of( AlarmSeverity.CRITICAL, new Condition("return powerConsumption >= 3000;", null, createDurationMs) ); - long clearDurationMs = 2000L; Condition clearRule = new Condition("return powerConsumption < 3000;", null, createDurationMs); - CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 3 seconds", + CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", arguments, createRules, clearRule); postTelemetry(deviceId, "{\"powerConsumption\":3500}"); Thread.sleep(createDurationMs - 2000); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index e0cc2ce764..5ca68d4e1b 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -104,7 +104,8 @@ public class GeofencingCalculatedFieldStateTest { ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); state = new GeofencingCalculatedFieldState(ctx.getEntityId()); - state.init(ctx); + state.setCtx(ctx, null); + state.init(); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 56fc2c1086..e46f3e1c15 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -87,7 +87,8 @@ public class ScriptCalculatedFieldStateTest { ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); state = new ScriptCalculatedFieldState(ctx.getEntityId()); - state.init(ctx); + state.setCtx(ctx, null); + state.init(); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 7a6109b5bf..df8bf1fbba 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -79,7 +79,8 @@ public class SimpleCalculatedFieldStateTest { ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getEntityId()); - state.init(ctx); + state.setCtx(ctx, null); + state.init(); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index bcbe52b5c9..ace27c08c1 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -595,7 +595,7 @@ public class TbRuleEngineQueueConsumerManagerTest { await().atMost(5, TimeUnit.SECONDS).until(() -> { for (TopicPartitionInfo partition : expectedPartitions) { if (consumers.stream().noneMatch(consumer -> consumer.subscribed && - consumer.pollingStarted && Set.of(partition).equals(consumer.getPartitions()))) { + consumer.pollingStarted && Set.of(partition).equals(consumer.getPartitions()))) { return false; } } @@ -605,7 +605,7 @@ public class TbRuleEngineQueueConsumerManagerTest { await().atMost(5, TimeUnit.SECONDS).until(() -> { return consumers.size() == 1 && consumers.stream() .anyMatch(consumer -> consumer.subscribed && consumer.pollingStarted && - expectedPartitions.equals(consumer.getPartitions())); + expectedPartitions.equals(consumer.getPartitions())); }); } Mockito.reset(ruleEngineConsumerContext.getSubmitStrategyFactory()); @@ -667,8 +667,8 @@ public class TbRuleEngineQueueConsumerManagerTest { return await().atMost(5, TimeUnit.SECONDS) .until(() -> consumers.stream() .filter(consumer -> consumer.getPartitions() != null && - consumer.getPartitions().size() == 1 && - consumer.getPartitions().contains(tpi)) + consumer.getPartitions().size() == 1 && + consumer.getPartitions().contains(tpi)) .findFirst().orElse(null), Objects::nonNull); } @@ -676,9 +676,9 @@ public class TbRuleEngineQueueConsumerManagerTest { return await().atMost(5, TimeUnit.SECONDS) .until(() -> consumers.stream() .filter(consumer -> consumer.getPartitions() != null && - consumer.getPartitions().size() == 1 && - consumer.getPartitions().stream() - .anyMatch(tpi -> tpi.getPartition().get().equals(partition))) + consumer.getPartitions().size() == 1 && + consumer.getPartitions().stream() + .anyMatch(tpi -> tpi.getPartition().get().equals(partition))) .findFirst().orElse(null), Objects::nonNull); } @@ -778,10 +778,6 @@ public class TbRuleEngineQueueConsumerManagerTest { return false; } - public Set getPartitions() { - return partitions; - } - public void setUpTestMsg() { testMsg = TbMsg.newMsg() .type(TbMsgType.POST_TELEMETRY_REQUEST) @@ -790,6 +786,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .data("{}") .build(); } + } } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java index 1106fad5b6..9bd9bb2e7a 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.ConsumerKey; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; @@ -191,6 +192,7 @@ public class TbRuleEngineStrategyTest { queue.setProcessingStrategy(processingStrategy); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue); + ConsumerKey consumerKey = new ConsumerKey(queueKey, null); var consumerManager = TbRuleEngineQueueConsumerManager.create() .ctx(ruleEngineConsumerContext) .queueKey(queueKey) @@ -238,7 +240,7 @@ public class TbRuleEngineStrategyTest { .map(this::toProto) .toList(); - consumerManager.processMsgs(protoMsgs, consumer, queueKey, queue); + consumerManager.processMsgs(protoMsgs, consumer, consumerKey, queue); processingData.forEach(data -> { verify(actorContext, times(data.attempts)).tell(argThat(msg -> diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index f9483965cc..3e1462b445 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -38,6 +38,8 @@ public interface TbQueueConsumer { boolean isStopped(); + Set getPartitions(); + List getFullTopicNames(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java index c2925d5ed6..0b0f34ad50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -20,7 +20,6 @@ import jakarta.validation.constraints.NotEmpty; import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; -import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.List; @@ -59,10 +58,4 @@ public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculat } - @Override - public boolean requiresScheduledReevaluation() { - return createRules.values().stream().anyMatch(rule -> rule.getCondition().getType() == AlarmConditionType.DURATION) || - (clearRule != null && clearRule.getCondition().getType() == AlarmConditionType.DURATION); - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..7b608192db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -72,8 +72,4 @@ public interface CalculatedFieldConfiguration { .collect(Collectors.toList()); } - default boolean requiresScheduledReevaluation() { - return false; - } - } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/CalculatedFieldStatePartitionRestoreMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/CalculatedFieldStatePartitionRestoreMsg.java new file mode 100644 index 0000000000..b16e2adb85 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/CalculatedFieldStatePartitionRestoreMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +@Data +public class CalculatedFieldStatePartitionRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final TopicPartitionInfo partition; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_PARTITION_RESTORE_MSG; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index e655011baa..c13b0200c7 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -138,6 +138,7 @@ public enum MsgType { CF_CACHE_INIT_MSG, // Sent to init caches for CF actor; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_STATE_PARTITION_RESTORE_MSG, CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index fac1116a30..4d99608a6d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1914,7 +1914,7 @@ message AlarmStateProto { message AlarmRuleStateProto { string severity = 1; - int64 lastEventTs = 2; - int64 duration = 3; - int64 eventCount = 4; + int64 eventCount = 2; + int64 firstEventTs = 3; + int64 lastEventTs = 4; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 7e7de64a5c..04fe2443ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -194,6 +194,11 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + @Override + public Set getPartitions() { + return partitions; + } + @Override public List getFullTopicNames() { if (partitions == null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index db5bac7170..b300e8c1b2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -25,6 +25,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.ConsumerKey; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import java.util.Collection; @@ -218,7 +219,7 @@ public class MainQueueConsumerManager consumer) { + private void consumerLoop(ConsumerKey consumerKey, TbQueueConsumer consumer) { try { while (!stopped && !consumer.isStopped()) { try { @@ -250,7 +251,7 @@ public class MainQueueConsumerManager msgs, TbQueueConsumer consumer, Object consumerKey, C config) throws Exception { + protected void processMsgs(List msgs, TbQueueConsumer consumer, ConsumerKey consumerKey, C config) throws Exception { log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, consumerKey, config); log.trace("Processed {} messages", msgs.size()); @@ -273,7 +274,7 @@ public class MainQueueConsumerManager { - void process(List msgs, TbQueueConsumer consumer, Object consumerKey, C config) throws Exception; + void process(List msgs, TbQueueConsumer consumer, ConsumerKey consumerKey, C config) throws Exception; } public interface ConsumerWrapper { @@ -285,6 +286,7 @@ public class MainQueueConsumerManager { + private final Map> consumers = new HashMap<>(); @Override @@ -307,8 +309,7 @@ public class MainQueueConsumerManager partitions, Consumer onStop, Function startOffsetProvider) { partitions.forEach(tpi -> { - Integer partitionId = tpi.getPartition().orElse(-1); - String key = queueKey + "-" + partitionId; + ConsumerKey key = new ConsumerKey(queueKey, tpi); Runnable callback = onStop != null ? () -> onStop.accept(tpi) : null; TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> { @@ -328,9 +329,11 @@ public class MainQueueConsumerManager> getConsumers() { return consumers.values(); } + } class SingleConsumerWrapper implements ConsumerWrapper { + private TbQueueConsumerTask consumer; @Override @@ -346,7 +349,7 @@ public class MainQueueConsumerManager(queueKey, () -> consumerCreator.apply(config, null), null); // no partitionId passed + consumer = new TbQueueConsumerTask<>(new ConsumerKey(queueKey, null), () -> consumerCreator.apply(config, null), null); // no partitionId passed } consumer.subscribe(partitions); if (!consumer.isRunning()) { @@ -361,5 +364,7 @@ public class MainQueueConsumerManager { @Getter - private final Object key; + private final ConsumerKey key; private volatile TbQueueConsumer consumer; private volatile Supplier> consumerSupplier; @Getter @@ -41,7 +41,7 @@ public class TbQueueConsumerTask { @Setter private Future task; - public TbQueueConsumerTask(Object key, Supplier> consumerSupplier, Runnable callback) { + public TbQueueConsumerTask(ConsumerKey key, Supplier> consumerSupplier, Runnable callback) { this.key = key; this.consumer = null; this.consumerSupplier = consumerSupplier; @@ -97,4 +97,18 @@ public class TbQueueConsumerTask { return task != null; } + public record ConsumerKey(Object queueKey, TopicPartitionInfo partition) { + + @Override + public String toString() { + if (partition != null) { + Integer partitionId = partition.getPartition().orElse(-1); + return queueKey + "-" + partitionId; + } else { + return queueKey.toString(); + } + } + + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java index be379fb76d..6bcc87af38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.queue.common.state; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Collections; +import java.util.Set; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; public class DefaultQueueStateService extends QueueStateService { @@ -26,4 +31,18 @@ public class DefaultQueueStateService partitions, RestoreCallback callback) { + if (callback != null) { + for (TopicPartitionInfo partition : partitions) { + callback.onPartitionRestored(partition); + } + callback.onAllPartitionsRestored(); + } + eventConsumer.addPartitions(partitions); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.addPartitions(withTopic(partitions, consumer.getTopic())); + } + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java index 2a38c9a86c..60cfe4c98f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java @@ -50,7 +50,7 @@ public class KafkaQueueStateService } @Override - protected void addPartitions(QueueKey queueKey, Set partitions, Runnable whenAllProcessed) { + protected void addPartitions(QueueKey queueKey, Set partitions, RestoreCallback callback) { Map eventsStartOffsets = eventsStartOffsetsProvider != null ? eventsStartOffsetsProvider.get() : null; // remembering the offsets before subscribing to states Set statePartitions = withTopic(partitions, stateConsumer.getTopic()); @@ -61,10 +61,13 @@ public class KafkaQueueStateService try { partitionsInProgress.remove(statePartition); log.info("Finished partition {} (still in progress: {})", statePartition, partitionsInProgress); + if (callback != null) { + callback.onPartitionRestored(statePartition); + } if (partitionsInProgress.isEmpty()) { log.info("All partitions processed"); - if (whenAllProcessed != null) { - whenAllProcessed.run(); + if (callback != null) { + callback.onAllPartitionsRestored(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java index e58d5eb036..e98e8dd7e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java @@ -49,7 +49,7 @@ public abstract class QueueStateService newPartitions, Runnable whenAllProcessed) { + public void update(QueueKey queueKey, Set newPartitions, RestoreCallback callback) { newPartitions = withTopic(newPartitions, eventConsumer.getTopic()); var writeLock = partitionsLock.writeLock(); writeLock.lock(); @@ -71,23 +71,15 @@ public abstract class QueueStateService partitions, Runnable whenAllProcessed) { - if (whenAllProcessed != null) { - whenAllProcessed.run(); - } - eventConsumer.addPartitions(partitions); - for (PartitionedQueueConsumerManager consumer : otherConsumers) { - consumer.addPartitions(withTopic(partitions, consumer.getTopic())); - } - } + protected abstract void addPartitions(QueueKey queueKey, Set partitions, RestoreCallback callback) ; protected void removePartitions(QueueKey queueKey, Set partitions) { eventConsumer.removePartitions(partitions); @@ -122,4 +114,12 @@ public abstract class QueueStateService implements TbQueueCon return stopped; } + @Override + public Set getPartitions() { + return partitions; + } + @Override public List getFullTopicNames() { return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); From ad042c4348b8464517c76ebf5b8ac3c30f025941 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 12:01:22 +0300 Subject: [PATCH 015/122] CF: add type filter to API --- .../controller/CalculatedFieldController.java | 27 ++++++++++++------- .../cf/DefaultTbCalculatedFieldService.java | 6 ++--- .../entitiy/cf/TbCalculatedFieldService.java | 4 ++- .../thingsboard/server/cf/AlarmRulesTest.java | 3 +++ .../server/controller/AbstractWebTest.java | 6 +++++ .../CalculatedFieldControllerTest.java | 13 +++++++++ .../server/dao/cf/CalculatedFieldService.java | 3 ++- .../dao/cf/BaseCalculatedFieldService.java | 14 ++++++++-- .../server/dao/cf/CalculatedFieldDao.java | 4 ++- .../dao/sql/cf/CalculatedFieldRepository.java | 6 ++--- .../dao/sql/cf/JpaCalculatedFieldDao.java | 9 ++++--- .../telemetry/TbCalculatedFieldsNode.java | 2 +- 12 files changed, 73 insertions(+), 24 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index c5b077c128..8c193a5b20 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -44,6 +44,7 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -159,19 +160,27 @@ public class CalculatedFieldController extends BaseController { ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @GetMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}) - public PageData getCalculatedFieldsByEntityId( - @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, - @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, - @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, - @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, - @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, - @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { + public PageData getCalculatedFieldsByEntityId(@Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) + @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) + @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @Parameter(description = "Calculated field type. If not specified, all types will be returned.") + @RequestParam(required = false) CalculatedFieldType type, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) + @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); checkParameter("entityId", entityIdStr); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); - return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); + return checkNotNull(tbCalculatedFieldService.findByTenantIdAndEntityId(getTenantId(), entityId, type, pageLink)); } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4dfaec91cf..42dc204119 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -22,6 +22,7 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -68,10 +69,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } @Override - public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { - TenantId tenantId = user.getTenantId(); + public PageData findByTenantIdAndEntityId(TenantId tenantId, EntityId entityId, CalculatedFieldType type, PageLink pageLink) { checkEntityExistence(tenantId, entityId); - return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + return calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId, type, pageLink); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 1e04a14a08..20705aaaff 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -16,9 +16,11 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; @@ -29,7 +31,7 @@ public interface TbCalculatedFieldService { CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); - PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + PageData findByTenantIdAndEntityId(TenantId tenantId, EntityId entityId, CalculatedFieldType type, PageLink pageLink); void delete(CalculatedField calculatedField, SecurityUser user); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index b0b28f06dc..d8c071f778 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -107,6 +108,8 @@ public class AlarmRulesTest extends AbstractControllerTest { Condition clearRule = new Condition("return temperature <= 25;", null, null); CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, clearRule); + assertThat(getCalculatedFields(deviceId, CalculatedFieldType.ALARM, new PageLink(1)).getData()) + .singleElement().isEqualTo(calculatedField); postTelemetry(deviceId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 42be12929d..2050e1075c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -93,6 +93,7 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -1333,6 +1334,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { return doPost("/api/calculatedField", calculatedField, CalculatedField.class); } + protected PageData getCalculatedFields(EntityId entityId, CalculatedFieldType type, PageLink pageLink) throws Exception { + return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields" + + (type != null ? "?type=" + type.name() + "&" : "?"), new TypeReference<>() {}, pageLink); + } + protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE, limit); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 27622b347a..9ca75c1b2c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoor import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.security.Authority; @@ -149,6 +150,18 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { .andExpect(status().isOk()); } + @Test + public void testGetCalculatedFields() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + calculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(getCalculatedFields(testDevice.getId(), null, new PageLink(10)).getData()) + .singleElement().isEqualTo(calculatedField); + assertThat(getCalculatedFields(testDevice.getId(), CalculatedFieldType.SIMPLE, new PageLink(10)).getData()) + .singleElement().isEqualTo(calculatedField); + } + @Test public void testDeleteCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 85cd8d24fd..e0481b6705 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; @@ -45,7 +46,7 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findCalculatedFieldsByTenantId(TenantId tenantId, PageLink pageLink); - PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + PageData findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, CalculatedFieldType type, PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index c0cb886747..254b557eb1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -35,8 +36,10 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; +import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -136,11 +139,18 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } @Override - public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + public PageData findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, CalculatedFieldType type, PageLink pageLink) { log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); validatePageLink(pageLink); - return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + Set types; + if (type == null) { + types = EnumSet.allOf(CalculatedFieldType.class); + types.remove(CalculatedFieldType.ALARM); + } else { + types = Set.of(type); + } + return calculatedFieldDao.findByEntityIdAndTypes(tenantId, entityId, types, pageLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index d5465cb8a1..40517fe78b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -24,6 +25,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; +import java.util.Set; public interface CalculatedFieldDao extends Dao { @@ -41,7 +43,7 @@ public interface CalculatedFieldDao extends Dao { PageData findAllByTenantId(TenantId tenantId, PageLink pageLink); - PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + PageData findByEntityIdAndTypes(TenantId tenantId, EntityId entityId, Set types, PageLink pageLink); List removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 7755ef036b..f2c3525a4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -38,9 +38,9 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); @Query("SELECT cf FROM CalculatedFieldEntity cf WHERE cf.tenantId = :tenantId " + - "AND cf.entityId = :entityId " + - "AND (:textSearch IS NULL OR ilike(cf.name, CONCAT('%', :textSearch, '%')) = true)") - Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, String textSearch, Pageable pageable); + "AND cf.entityId = :entityId AND cf.type IN :types " + + "AND (:textSearch IS NULL OR ilike(cf.name, CONCAT('%', :textSearch, '%')) = true)") + Page findByTenantIdAndEntityIdAndTypes(UUID tenantId, UUID entityId, List types, String textSearch, Pageable pageable); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 385839dded..7a8c914e74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +35,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; +import java.util.Set; import java.util.UUID; @Slf4j @@ -83,9 +85,10 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { - log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); - return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); + public PageData findByEntityIdAndTypes(TenantId tenantId, EntityId entityId, Set types, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId [{}] and type [{}] and pageLink [{}]", entityId, types, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findByTenantIdAndEntityIdAndTypes(tenantId.getId(), entityId.getId(), + types.stream().map(Enum::name).toList(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java index e703e9dd25..049aa9e758 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -44,7 +44,7 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; @Slf4j @RuleNode( type = ComponentType.ACTION, - name = "calculated fields", + name = "calculated fields", // TODO: rename to "alarms and calculated fields" configClazz = EmptyNodeConfiguration.class, nodeDescription = "Pushes incoming messages to calculated fields service", nodeDetails = "Node enables the processing of calculated fields without persisting incoming messages to the database. " + From 2009682642830a549019f4c6469659faf90104aa Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 12:02:13 +0300 Subject: [PATCH 016/122] Make BuildProperties lazy-loaded --- .../thingsboard/server/service/install/ProjectInfo.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java b/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java index 4422d952a5..8b7218a981 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java @@ -19,14 +19,17 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.info.BuildProperties; import org.springframework.stereotype.Component; +import java.util.Optional; + @Component @RequiredArgsConstructor public class ProjectInfo { - private final BuildProperties buildProperties; + private final Optional buildProperties; public String getProjectVersion() { - return buildProperties.getVersion().replaceAll("[^\\d.]", ""); + return buildProperties.orElseThrow(() -> new IllegalStateException("Build properties are missing. Please rebuild the project with maven")) + .getVersion().replaceAll("[^\\d.]", ""); } public String getProductType() { From 6a83ea56365fb1e53119b12f09dafb2735203f42 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 12:38:41 +0300 Subject: [PATCH 017/122] Rename "calculated fields" node to "calculated fields and alarm rules" --- .../controller/CalculatedFieldControllerTest.java | 10 +++++----- .../server/dao/cf/BaseCalculatedFieldService.java | 7 ++++--- .../rule/engine/telemetry/TbCalculatedFieldsNode.java | 10 +++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 9ca75c1b2c..81c7c6623e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration; -import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; @@ -176,13 +176,13 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); } - private CalculatedField getCalculatedField(DeviceId deviceId) { - return getCalculatedField(deviceId, getSimpleCalculatedFieldConfig()); + private CalculatedField getCalculatedField(EntityId entityId) { + return getCalculatedField(entityId, getSimpleCalculatedFieldConfig()); } - private CalculatedField getCalculatedField(DeviceId deviceId, CalculatedFieldConfiguration configuration) { + private CalculatedField getCalculatedField(EntityId entityId, CalculatedFieldConfiguration configuration) { CalculatedField calculatedField = new CalculatedField(); - calculatedField.setEntityId(deviceId); + calculatedField.setEntityId(entityId); calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 254b557eb1..2efa32215a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -34,7 +34,8 @@ import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.IncorrectParameterException; -import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.service.validator.CalculatedFieldDataValidator; +import org.thingsboard.server.dao.service.validator.CalculatedFieldLinkDataValidator; import java.util.EnumSet; import java.util.List; @@ -55,8 +56,8 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private final CalculatedFieldDao calculatedFieldDao; private final CalculatedFieldLinkDao calculatedFieldLinkDao; - private final DataValidator calculatedFieldDataValidator; - private final DataValidator calculatedFieldLinkDataValidator; + private final CalculatedFieldDataValidator calculatedFieldDataValidator; + private final CalculatedFieldLinkDataValidator calculatedFieldLinkDataValidator; @Override public CalculatedField save(CalculatedField calculatedField) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java index 049aa9e758..7c6d1884bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -44,13 +44,13 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; @Slf4j @RuleNode( type = ComponentType.ACTION, - name = "calculated fields", // TODO: rename to "alarms and calculated fields" + name = "calculated fields and alarm rules", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Pushes incoming messages to calculated fields service", - nodeDetails = "Node enables the processing of calculated fields without persisting incoming messages to the database. " + - "By default, the processing of calculated fields is triggered by the save attributes and save time series nodes. " + + nodeDescription = "Pushes incoming messages to calculated fields and alarm rules services", + nodeDetails = "Node enables the processing of calculated fields and alarm rules without persisting incoming messages to the database. " + + "By default, the processing of calculated fields and alarm rules is triggered by the save attributes and save time series nodes. " + "This rule node accepts the same messages as these nodes but allows you to trigger the processing of calculated " + - "fields independently, ensuring that derived data can be computed and utilized in real time without storing the original message in the database.", + "fields or alarm rules independently, ensuring that derived data can be computed and utilized in real time without storing the original message in the database.", configDirective = "tbNodeEmptyConfig", icon = "published_with_changes" ) From b21b046918ef4c762ac2b6e608c2b9794ea89ccf Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 12:39:08 +0300 Subject: [PATCH 018/122] Rename "Calculated fields" save strategy to "Calculated fields and alarm rules" --- .../rule-node/action/advanced-processing-setting.component.html | 2 +- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html index edce8b12a9..a542d3e07d 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html @@ -39,6 +39,6 @@ > diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f78e95e56d..775f20cb5a 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5505,7 +5505,7 @@ "time-series": "Time series", "latest": "Latest values", "web-sockets": "WebSockets", - "calculated-fields": "Calculated fields" + "calculated-fields-and-alarm-rules": "Calculated fields and alarm rules" }, "save-attribute": { "processing-settings": "Processing settings", From bbbcc583c572750a185b84ed5a739ba543722ca4 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Mon, 6 Oct 2025 13:29:07 +0300 Subject: [PATCH 019/122] Added support for arguments only propagation mode --- ...tractCalculatedFieldProcessingService.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 16 ++++++ .../cf/ctx/state/CalculatedFieldCtx.java | 23 +++++--- .../cf/ctx/state/CalculatedFieldState.java | 4 +- .../ctx/state/SimpleCalculatedFieldState.java | 11 +--- .../GeofencingCalculatedFieldState.java | 8 +-- .../propagation/PropagationArgumentEntry.java | 1 + .../PropagationCalculatedFieldState.java | 53 ++++++++++++++++--- ...opagationCalculatedFieldConfiguration.java | 15 ++++++ 9 files changed, 102 insertions(+), 31 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 232476c15a..f5ec782992 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -88,7 +88,7 @@ public abstract class AbstractCalculatedFieldProcessingService { protected abstract String getExecutorNamePrefix(); protected ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - Map> argFutures = switch (ctx.getCalculatedField().getType()) { + Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); }; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d027a2f9fe..c6af49bf4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; @@ -105,6 +107,20 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected void validateNewEntry(String key, ArgumentEntry newEntry) {} + protected ObjectNode toSimpleResult(boolean useLatestTs, ObjectNode valuesNode) { + if (!useLatestTs) { + return valuesNode; + } + long latestTs = getLatestTimestamp(); + if (latestTs == -1) { + return valuesNode; + } + ObjectNode resultNode = JacksonUtil.newObjectNode(); + resultNode.put("ts", latestTs); + resultNode.set("values", valuesNode); + return resultNode; + } + private void updateLastUpdateTimestamp(ArgumentEntry entry) { long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index ece459350f..339ba40838 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -103,6 +103,7 @@ public class CalculatedFieldCtx { private long scheduledUpdateIntervalMillis; private Argument propagationArgument; + private boolean applyExpressionForResolvedArguments; public CalculatedFieldCtx(CalculatedField calculatedField, ActorSystemContext systemContext) { @@ -159,6 +160,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) { propagationArgument = propagationConfig.toPropagationArgument(); + applyExpressionForResolvedArguments = propagationConfig.isApplyExpressionToResolvedArguments(); relationQueryDynamicArguments = true; } } @@ -177,13 +179,13 @@ public class CalculatedFieldCtx { public void init() { switch (cfType) { - case SCRIPT, PROPAGATION -> { - try { - initTbelExpression(expression); - initialized = true; - } catch (Exception e) { - throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + case SCRIPT -> initTbelExpression(); + case PROPAGATION -> { + if (applyExpressionForResolvedArguments) { + initTbelExpression(); + return; } + initialized = true; } case GEOFENCING -> initialized = true; case SIMPLE -> { @@ -206,6 +208,15 @@ public class CalculatedFieldCtx { } } + private void initTbelExpression() { + try { + initTbelExpression(expression); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } + } + public double evaluateSimpleExpression(String expressionStr, CalculatedFieldState state) { Expression expression = simpleExpressions.get(expressionStr).get(); for (Map.Entry entry : state.getArguments().entrySet()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index ad8005af83..ecd8b0d0f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; import java.util.Map; @@ -36,7 +37,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldState.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldState.class, name = "ALARM"), + @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION") }) public interface CalculatedFieldState { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 3a98fee361..bd5ec720df 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -84,16 +84,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } else { valuesNode.set(outputName, JacksonUtil.valueToTree(result)); } - - long latestTs = getLatestTimestamp(); - if (useLatestTs && latestTs != -1) { - ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", latestTs); - resultNode.set("values", valuesNode); - return resultNode; - } else { - return valuesNode; - } + return toSimpleResult(useLatestTs, valuesNode); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java index f3bf8750cf..da1acc1905 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java @@ -172,13 +172,7 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { } private JsonNode toResultNode(OutputType outputType, ObjectNode valuesNode) { - if (OutputType.ATTRIBUTES.equals(outputType) || latestTimestamp == -1) { - return valuesNode; - } - ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", latestTimestamp); - resultNode.set("values", valuesNode); - return resultNode; + return toSimpleResult(outputType == OutputType.TIME_SERIES, valuesNode); } private GeofencingEvalResult aggregateZoneGroup(List zoneResults) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java index c7d49a4d40..f09ae93999 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java @@ -32,6 +32,7 @@ public class PropagationArgumentEntry implements ArgumentEntry { private boolean forceResetPrevious; + // TODO: do we need to persist this? public PropagationArgumentEntry(List propagationEntityIds) { this.propagationEntityIds = propagationEntityIds; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java index 21a4493c91..9554b28024 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -15,10 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state.propagation; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; @@ -26,6 +30,7 @@ import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; @@ -37,6 +42,12 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState super(entityId); } + @Override + public void init(CalculatedFieldCtx ctx) { + super.init(ctx); + requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.PROPAGATION; @@ -48,12 +59,42 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry) || propagationArgumentEntry.isEmpty()) { return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build()); } - return Futures.transform(super.performCalculation(updatedArgs, ctx), telemetryCfResult -> - PropagationCalculatedFieldResult.builder() - .propagationEntityIds(propagationArgumentEntry.getPropagationEntityIds()) - .result((TelemetryCalculatedFieldResult) telemetryCfResult) - .build(), - MoreExecutors.directExecutor()); + if (ctx.isApplyExpressionForResolvedArguments()) { + return Futures.transform(super.performCalculation(updatedArgs, ctx), telemetryCfResult -> + PropagationCalculatedFieldResult.builder() + .propagationEntityIds(propagationArgumentEntry.getPropagationEntityIds()) + .result((TelemetryCalculatedFieldResult) telemetryCfResult) + .build(), + MoreExecutors.directExecutor()); + } + return Futures.immediateFuture(PropagationCalculatedFieldResult.builder() + .propagationEntityIds(propagationArgumentEntry.getPropagationEntityIds()) + .result(toTelemetryResult(ctx)) + .build()); + } + + private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx) { + Output output = ctx.getOutput(); + TelemetryCalculatedFieldResult.TelemetryCalculatedFieldResultBuilder telemetryCfBuilder = + TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()); + ObjectNode valuesNode = JacksonUtil.newObjectNode(); + arguments.forEach((argumentName, argumentEntry) -> { + if (argumentEntry instanceof PropagationArgumentEntry) { + return; + } + if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) { + // TODO: use argumentName as a key or no? + JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), argumentName); + return; + } + throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + argumentName + ". " + + "Only Latest telemetry or Attribute arguments supported for 'Arguments Only' propagation mode!"); + }); + ObjectNode result = toSimpleResult(output.getType() == OutputType.TIME_SERIES, valuesNode); + telemetryCfBuilder.result(result); + return telemetryCfBuilder.build(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index 7585c30438..b592264d6c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -33,6 +33,8 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField private EntitySearchDirection direction; private String relationType; + private boolean applyExpressionToResolvedArguments; + @Override public CalculatedFieldType getType() { return CalculatedFieldType.PROPAGATION; @@ -48,6 +50,19 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField if (StringUtils.isBlank(relationType)) { throw new IllegalArgumentException("Propagation calculated field relation type must be specified!"); } + if (!applyExpressionToResolvedArguments) { + arguments.forEach((name, argument) -> { + if (argument.getRefEntityKey() == null) { + throw new IllegalArgumentException("Argument: '" + name + "' doesn't have reference entity key configured!"); + } + if (argument.getRefEntityKey().getType() == ArgumentType.TS_ROLLING) { + throw new IllegalArgumentException("Argument type: 'Time series rolling' detected for argument: '" + name + "'! " + + "Only 'Attribute' or 'Latest telemetry' arguments are allowed for in 'Arguments only' propagation mode!"); + } + }); + } else if (StringUtils.isBlank(expression)) { + throw new IllegalArgumentException("Expression must be specified for 'Expression result' propagation mode!"); + } } public Argument toPropagationArgument() { From 60ef35ac8ec193dd5cce75fc66875f8ed1adb667 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 13:21:47 +0300 Subject: [PATCH 020/122] Alarm rules for customer entity --- .../DefaultCalculatedFieldQueueService.java | 9 +- .../server/service/cf/OwnerService.java | 1 + .../cf/DefaultTbCalculatedFieldService.java | 7 +- .../DefaultTelemetrySubscriptionService.java | 1 - .../thingsboard/server/cf/AlarmRulesTest.java | 106 ++++++++++++++---- .../rule/condition/AlarmConditionValue.java | 4 + .../common/data/cf/CalculatedField.java | 11 ++ 7 files changed, 104 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index ab06349e3d..a3e049d1e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -25,6 +25,7 @@ import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -45,9 +46,7 @@ import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; -import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.function.Predicate; import java.util.function.Supplier; @@ -73,10 +72,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; - private static final Set supportedReferencedEntities = EnumSet.of( - EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT - ); - @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); @@ -155,7 +150,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { - if (!supportedReferencedEntities.contains(entityId.getEntityType())) { + if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java index 84c46715e7..8e2bcecce2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java @@ -44,6 +44,7 @@ public class OwnerService { return switch (entityId.getEntityType()) { case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId).getOwnerId(); case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId).getOwnerId(); + case CUSTOMER -> tenantId; default -> throw new UnsupportedOperationException(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 42dc204119..460bca52bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -96,10 +96,11 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private void checkEntityExistence(TenantId tenantId, EntityId entityId) { - switch (entityId.getEntityType()) { - case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + if (CalculatedField.SUPPORTED_ENTITIES.contains(entityId.getEntityType())) { + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); - default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } else { + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index cd5848ad0d..69b41addf9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -169,7 +169,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(resultFuture, result -> { if (strategy.processCalculatedFields()) { - // TODO: divide CFs and alarm rules processing calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); } else { request.getCallback().onSuccess(null); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index d8c071f778..f2d15e4039 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.cf; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +41,8 @@ import org.thingsboard.server.common.data.alarm.rule.condition.expression.Simple import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate.NumericOperation; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate.StringOperation; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; @@ -53,7 +54,6 @@ import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.page.PageLink; @@ -81,14 +81,14 @@ public class AlarmRulesTest extends AbstractControllerTest { private EventDao eventDao; private Device device; - private DeviceId deviceId; + private EntityId originatorId; private EventId latestEventId; @Before public void beforeEach() throws Exception { loginTenantAdmin(); device = createDevice("Device A", "aaa"); - deviceId = device.getId(); + originatorId = device.getId(); } @Test @@ -106,33 +106,33 @@ public class AlarmRulesTest extends AbstractControllerTest { ); Condition clearRule = new Condition("return temperature <= 25;", null, null); - CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", arguments, createRules, clearRule); - assertThat(getCalculatedFields(deviceId, CalculatedFieldType.ALARM, new PageLink(1)).getData()) + assertThat(getCalculatedFields(originatorId, CalculatedFieldType.ALARM, new PageLink(1)).getData()) .singleElement().isEqualTo(calculatedField); - postTelemetry(deviceId, "{\"temperature\":50}"); + postTelemetry(originatorId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(deviceId, "{\"temperature\":100}"); + postTelemetry(originatorId, "{\"temperature\":100}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isSeverityUpdated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(deviceId, "{\"temperature\":101}"); + postTelemetry(originatorId, "{\"temperature\":101}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isUpdated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(deviceId, "{\"temperature\":20}"); + postTelemetry(originatorId, "{\"temperature\":20}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCleared()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -164,10 +164,10 @@ public class AlarmRulesTest extends AbstractControllerTest { AlarmSeverity.CRITICAL, new Condition(simpleExpression, null, null) ); - CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", arguments, createRules, null); - postTelemetry(deviceId, "{\"temperature\":100}"); + postTelemetry(originatorId, "{\"temperature\":100}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -176,7 +176,7 @@ public class AlarmRulesTest extends AbstractControllerTest { } /* - * todo: state restore (event count) + * todo: test state restore (event count) * */ @Test public void testCreateAlarmForRepeatingCondition() throws Exception { @@ -194,14 +194,14 @@ public class AlarmRulesTest extends AbstractControllerTest { AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", eventsCountCritical, null) ); - CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", arguments, createRules, null); for (int i = 0; i < 4; i++) { - postTelemetry(deviceId, "{\"temperature\":50}"); + postTelemetry(originatorId, "{\"temperature\":50}"); Thread.sleep(10); } assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); - postTelemetry(deviceId, "{\"temperature\":50}"); + postTelemetry(originatorId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); @@ -210,7 +210,7 @@ public class AlarmRulesTest extends AbstractControllerTest { }); for (int i = 0; i < 5; i++) { - postTelemetry(deviceId, "{\"temperature\":50}"); + postTelemetry(originatorId, "{\"temperature\":50}"); Thread.sleep(10); } checkAlarmResult(calculatedField, alarmResult -> { @@ -236,9 +236,9 @@ public class AlarmRulesTest extends AbstractControllerTest { ); Condition clearRule = new Condition("return powerConsumption < 3000;", null, createDurationMs); - CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", + CalculatedField calculatedField = createAlarmCf(originatorId, "High power consumption during 5 seconds", arguments, createRules, clearRule); - postTelemetry(deviceId, "{\"powerConsumption\":3500}"); + postTelemetry(originatorId, "{\"powerConsumption\":3500}"); Thread.sleep(createDurationMs - 2000); assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); @@ -272,11 +272,11 @@ public class AlarmRulesTest extends AbstractControllerTest { device.setCustomerId(customerId); device = doPost("/api/device", device, Device.class); - CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", arguments, createRules, null); postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"temperatureThreshold\":50}"); - postTelemetry(deviceId, "{\"temperature\":51}"); + postTelemetry(originatorId, "{\"temperature\":51}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -284,6 +284,53 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + @Test + public void testCreateAndClearAlarm_customerAlarmRule_simpleExpression() throws Exception { + Argument locationArgument = new Argument(); + locationArgument.setRefEntityKey(new ReferencedEntityKey("location", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + locationArgument.setDefaultValue("unknown"); + + Argument locationFilterArgument = new Argument(); + locationFilterArgument.setRefEntityKey(new ReferencedEntityKey("locationFilter", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + locationFilterArgument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + locationFilterArgument.setDefaultValue("None"); + loginSysAdmin(); + postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"locationFilter\":\"Kyiv\"}"); + loginTenantAdmin(); + + Map arguments = Map.of( + "location", locationArgument, + "locationFilter", locationFilterArgument + ); + originatorId = customerId; + + Map createRules = Map.of( + AlarmSeverity.INDETERMINATE, new Condition(createSimpleExpression( + "location", StringOperation.CONTAINS, new AlarmConditionValue<>(null, "locationFilter") + ), null, null) + ); + Condition clearRule = new Condition(createSimpleExpression( + "location", StringOperation.NOT_CONTAINS, new AlarmConditionValue<>(null, "locationFilter") + ), null, null); + + CalculatedField calculatedField = createAlarmCf(customerId, "New resident", + arguments, createRules, clearRule); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"location\":\"Ukraine, Kyiv\"}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.INDETERMINATE); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + + postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"location\":\"Ukraine, Lviv\"}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCleared()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.INDETERMINATE); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.CLEARED_UNACK); + }); + } + private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { TbAlarmResult alarmResult = getLatestAlarmResult(calculatedField.getId()); @@ -291,7 +338,7 @@ public class AlarmRulesTest extends AbstractControllerTest { assertion.accept(alarmResult); Alarm alarm = alarmResult.getAlarm(); - assertThat(alarm.getOriginator()).isEqualTo(deviceId); + assertThat(alarm.getOriginator()).isEqualTo(originatorId); assertThat(alarm.getType()).isEqualTo(calculatedField.getName()); }); } @@ -303,8 +350,7 @@ public class AlarmRulesTest extends AbstractControllerTest { } CalculatedFieldDebugEvent debugEvent = debugEvents.get(0); if (debugEvent.getError() != null) { - System.err.println("CF error: " + debugEvent.getError()); - Assertions.fail(); + throw new RuntimeException(debugEvent.getError()); } if (debugEvent.getId().equals(latestEventId)) { return null; @@ -381,6 +427,18 @@ public class AlarmRulesTest extends AbstractControllerTest { return rule; } + private SimpleAlarmConditionExpression createSimpleExpression(String argument, StringOperation stringOperation, AlarmConditionValue conditionValue) { + SimpleAlarmConditionExpression simpleExpression = new SimpleAlarmConditionExpression(); + AlarmConditionFilter filter = new AlarmConditionFilter(); + filter.setArgument(argument); + StringFilterPredicate predicate = new StringFilterPredicate(); + predicate.setOperation(stringOperation); + predicate.setValue(conditionValue); + filter.setPredicate(predicate); + simpleExpression.setFilters(List.of(filter)); + return simpleExpression; + } + private List getDebugEvents(CalculatedFieldId calculatedFieldId, int limit) { return eventDao.findLatestEvents(tenantId.getId(), calculatedFieldId.getId(), EventType.DEBUG_CALCULATED_FIELD, limit).stream() .map(e -> (CalculatedFieldDebugEvent) e).toList(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java index 4bde76820a..84a1498ef6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java @@ -15,9 +15,13 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor public class AlarmConditionValue { private T staticValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 9dd92294db..f39ce27169 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -39,6 +40,9 @@ import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; import java.io.Serial; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; @Schema @Data @@ -48,6 +52,13 @@ public class CalculatedField extends BaseData implements HasN @Serial private static final long serialVersionUID = 4491966747773381420L; + public static final Set SUPPORTED_ENTITIES = Collections.unmodifiableSet(EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE, EntityType.CUSTOMER + )); + public static final Set SUPPORTED_REFERENCED_ENTITIES = Collections.unmodifiableSet(EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + )); + private TenantId tenantId; private EntityId entityId; From e8caf189682641dbe1e60e39bd29c74003b02b21 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Mon, 6 Oct 2025 15:33:21 +0300 Subject: [PATCH 021/122] Resolved TODOs & removed propagation argument persistence logic --- .../propagation/PropagationArgumentEntry.java | 1 - .../PropagationCalculatedFieldState.java | 3 +- .../server/utils/CalculatedFieldUtils.java | 14 ------ .../utils/CalculatedFieldUtilsTest.java | 44 +++++++++++++++++++ common/proto/src/main/proto/queue.proto | 1 - 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java index f09ae93999..c7d49a4d40 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java @@ -32,7 +32,6 @@ public class PropagationArgumentEntry implements ArgumentEntry { private boolean forceResetPrevious; - // TODO: do we need to persist this? public PropagationArgumentEntry(List propagationEntityIds) { this.propagationEntityIds = propagationEntityIds; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java index 9554b28024..d039d7aa7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -85,8 +85,7 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState return; } if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) { - // TODO: use argumentName as a key or no? - JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), argumentName); + JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue()); return; } throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + argumentName + ". " + diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 51ca44fd54..330dc89352 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -32,7 +32,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.EntityIdProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; @@ -51,10 +50,8 @@ import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; -import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -62,8 +59,6 @@ import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; - public class CalculatedFieldUtils { public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { @@ -102,7 +97,6 @@ public class CalculatedFieldUtils { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); - case PROPAGATION -> builder.addAllPropagationEntityIds(toPropagationEntityIdsProto((PropagationArgumentEntry) argEntry)); } }); if (state instanceof AlarmCalculatedFieldState alarmState) { @@ -117,10 +111,6 @@ public class CalculatedFieldUtils { return builder.build(); } - private static List toPropagationEntityIdsProto(PropagationArgumentEntry argEntry) { - return argEntry.getPropagationEntityIds().stream().map(ProtoUtils::toProto).collect(Collectors.toList()); - } - private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) { return AlarmRuleStateProto.newBuilder() .setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse("")) @@ -214,10 +204,6 @@ public class CalculatedFieldUtils { alarmState.getCreateRuleStates().put(severity, ruleState); } } - case PROPAGATION -> { - List propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList(); - state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds)); - } } return state; diff --git a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java index 40a7a14e1c..0d57f90206 100644 --- a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java @@ -26,24 +26,31 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.mock; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + @ExtendWith(MockitoExtension.class) class CalculatedFieldUtilsTest { @@ -105,4 +112,41 @@ class CalculatedFieldUtilsTest { assertThat(fromProtoGeoArgument.getZoneStates().get(z2).getLastPresence()).isNull(); } + @Test + void toProtoAndFromProto_shouldCreatePropagationStateWithoutPropagationArgument() { + // given + CalculatedFieldEntityCtxId stateId = mock(CalculatedFieldEntityCtxId.class); + given(stateId.tenantId()).willReturn(TENANT_ID); + given(stateId.cfId()).willReturn(CF_ID); + given(stateId.entityId()).willReturn(DEVICE_ID); + + AssetId propagationAssetId = new AssetId(UUID.fromString("17bbf99c-3b87-4d21-b07d-da7409bb2bb7")); + PropagationArgumentEntry propagationArgumentEntry = new PropagationArgumentEntry(List.of(propagationAssetId)); + + long lastUpdateTs = System.currentTimeMillis(); + SingleValueArgumentEntry singleValueArgumentEntry = new SingleValueArgumentEntry(new BaseAttributeKvEntry(new StringDataEntry("state", "active"), lastUpdateTs, 1L)); + + CalculatedFieldCtx cfCtxMock = mock(CalculatedFieldCtx.class); + + CalculatedFieldState state = new PropagationCalculatedFieldState(DEVICE_ID); + state.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry, "state", singleValueArgumentEntry), cfCtxMock); + + // when + CalculatedFieldStateProto proto = toProto(stateId, state); + + // then + CalculatedFieldState restored = CalculatedFieldUtils.fromProto(stateId, proto); + + // Propagation argument is not persisted -> should be absent after restore + assertThat(restored).isNotNull(); + assertThat(restored).isInstanceOf(PropagationCalculatedFieldState.class); + + PropagationCalculatedFieldState propagationState = (PropagationCalculatedFieldState) restored; + + assertThat(propagationState.getEntityId()).isEqualTo(DEVICE_ID); + assertThat(propagationState.getArguments()).isNotNull(); + assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isNull(); + assertThat(propagationState.getArguments().get("state")).isNotNull().isEqualTo(singleValueArgumentEntry); + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1e3e121202..fac1116a30 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -922,7 +922,6 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - repeated EntityIdProto propagationEntityIds = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 49424a553f737012a3e794c8c6959a507c77c0ae Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 6 Oct 2025 15:40:32 +0300 Subject: [PATCH 022/122] Alarm rules CF: fix owner entities cache for customers --- ...alculatedFieldManagerMessageProcessor.java | 91 +++++++++---------- .../server/actors/tenant/TenantActor.java | 2 +- .../server/service/cf/OwnerService.java | 20 ++-- .../entitiy/EntityStateSourcingListener.java | 4 + .../cf/DefaultTbCalculatedFieldService.java | 21 +++-- .../queue/DefaultTbClusterService.java | 12 +++ .../processing/AbstractConsumerService.java | 9 +- .../thingsboard/server/cf/AlarmRulesTest.java | 45 ++++----- .../server/cluster/TbClusterService.java | 3 + .../common/data/cf/CalculatedField.java | 12 ++- .../common/data/cf/CalculatedFieldType.java | 10 +- .../dao/customer/CustomerServiceImpl.java | 12 ++- 12 files changed, 145 insertions(+), 96 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 5e7437d12d..d29fab508b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; @@ -45,6 +46,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; @@ -84,6 +86,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final CalculatedFieldService cfDaoService; private final DeviceService deviceService; private final AssetService assetService; + private final CustomerService customerService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; @@ -100,6 +103,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.cfDaoService = systemContext.getCalculatedFieldService(); this.deviceService = systemContext.getDeviceService(); this.assetService = systemContext.getAssetService(); + this.customerService = systemContext.getCustomerService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); @@ -150,56 +154,29 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var entityType = msg.getData().getEntityId().getEntityType(); var event = msg.getData().getEvent(); switch (entityType) { - case CALCULATED_FIELD: { + case CALCULATED_FIELD -> { switch (event) { - case CREATED: - onCfCreated(msg.getData(), msg.getCallback()); - break; - case UPDATED: - onCfUpdated(msg.getData(), msg.getCallback()); - break; - case DELETED: - onCfDeleted(msg.getData(), msg.getCallback()); - break; - default: - msg.getCallback().onSuccess(); - break; + case CREATED -> onCfCreated(msg.getData(), msg.getCallback()); + case UPDATED -> onCfUpdated(msg.getData(), msg.getCallback()); + case DELETED -> onCfDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); } - break; } - case DEVICE: - case ASSET: { + case DEVICE, ASSET, CUSTOMER -> { switch (event) { - case CREATED: - onEntityCreated(msg.getData(), msg.getCallback()); - break; - case UPDATED: - onEntityUpdated(msg.getData(), msg.getCallback()); - break; - case DELETED: - onEntityDeleted(msg.getData(), msg.getCallback()); - break; - default: - msg.getCallback().onSuccess(); - break; + case CREATED -> onEntityCreated(msg.getData(), msg.getCallback()); + case UPDATED -> onEntityUpdated(msg.getData(), msg.getCallback()); + case DELETED -> onEntityDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); } - break; } - case DEVICE_PROFILE: - case ASSET_PROFILE: { + case DEVICE_PROFILE, ASSET_PROFILE -> { switch (event) { - case DELETED: - onProfileDeleted(msg.getData(), msg.getCallback()); - break; - default: - msg.getCallback().onSuccess(); - break; + case DELETED -> onProfileDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); } - break; - } - default: { - msg.getCallback().onSuccess(); } + default -> msg.getCallback().onSuccess(); } } @@ -271,7 +248,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - entityProfileCache.removeEntityId(msg.getEntityId()); + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); @@ -404,9 +384,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } // process all cfs related to owner entity if (entityId.getEntityType().isOneOf(EntityType.TENANT, EntityType.CUSTOMER)) { - List ownerCFs = filterOwnerEntitiesCFs(msg); - if (!ownerCFs.isEmpty()) { - cfExecService.pushMsgToLinks(msg, ownerCFs, callback); + List ownedEntitiesCFs = filterOwnedEntitiesCFs(msg); + if (!ownedEntitiesCFs.isEmpty()) { + cfExecService.pushMsgToLinks(msg, ownedEntitiesCFs, callback); } else { callback.onSuccess(); } @@ -473,8 +453,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private List filterOwnerEntitiesCFs(CalculatedFieldTelemetryMsg msg) { - Set entities = getOwnerEntities(msg.getEntityId()); + private List filterOwnedEntitiesCFs(CalculatedFieldTelemetryMsg msg) { + Set entities = getOwnedEntities(msg.getEntityId()); var proto = msg.getProto(); List result = new ArrayList<>(); for (var entityId : entities) { @@ -516,7 +496,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private Set getOwnerEntities(EntityId entityId) { + private Set getOwnedEntities(EntityId entityId) { if (entityId == null) { return Collections.emptySet(); } @@ -627,21 +607,32 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.trace("Processing device record: {}", idInfo); try { entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); - ownerEntities.computeIfAbsent(idInfo.getOwnerId(), ownerId -> new HashSet<>()).add(idInfo.getEntityId()); + ownerEntities.computeIfAbsent(idInfo.getOwnerId(), __ -> new HashSet<>()).add(idInfo.getEntityId()); } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } } + PageDataIterable assetIdInfos = new PageDataIterable<>(pageLink -> assetService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); - ownerEntities.computeIfAbsent(idInfo.getOwnerId(), ownerId -> new HashSet<>()).add(idInfo.getEntityId()); + ownerEntities.computeIfAbsent(idInfo.getOwnerId(), __ -> new HashSet<>()).add(idInfo.getEntityId()); } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } } + + PageDataIterable customers = new PageDataIterable<>(pageLink -> customerService.findCustomersByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); + for (Customer customer : customers) { + log.trace("Processing customer record: {}", customer); + try { + ownerEntities.computeIfAbsent(customer.getTenantId(), __ -> new HashSet<>()).add(customer.getId()); + } catch (Exception e) { + log.error("Failed to process customer record: {}", customer, e); + } + } } private void updateEntityOwner(EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index e33965a42e..35a7f01b2e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -350,7 +350,7 @@ public class TenantActor extends RuleChainManagerActor { } } if (cfActor != null) { - if (msg.getEntityId().getEntityType().isOneOf(EntityType.CALCULATED_FIELD, EntityType.DEVICE, EntityType.ASSET)) { + if (msg.getEntityId().getEntityType().isOneOf(EntityType.CALCULATED_FIELD, EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER)) { cfActor.tellWithHighPriority(new CalculatedFieldEntityLifecycleMsg(tenantId, msg)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java index 8e2bcecce2..269d90e5e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/OwnerService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.EntityType; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; import java.util.HashSet; @@ -39,6 +41,7 @@ public class OwnerService { private final DeviceService deviceService; private final AssetService assetService; + private final CustomerService customerService; public EntityId getOwner(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { @@ -50,19 +53,24 @@ public class OwnerService { } public Set getOwnedEntities(TenantId tenantId, EntityId ownerId) { - Set ownerEntities = new HashSet<>(); + Set ownedEntities = new HashSet<>(); if (EntityType.CUSTOMER.equals(ownerId.getEntityType())) { PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByFilter(DeviceInfoFilter.builder().tenantId(tenantId).customerId((CustomerId) ownerId).build(), pageLink), 1000); - deviceIdInfos.forEach(deviceInfo -> ownerEntities.add(deviceInfo.getId())); + deviceIdInfos.forEach(deviceInfo -> ownedEntities.add(deviceInfo.getId())); + PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantIdAndCustomerId(tenantId, (CustomerId) ownerId, pageLink), 1000); - assets.forEach(asset -> ownerEntities.add(asset.getId())); + assets.forEach(asset -> ownedEntities.add(asset.getId())); } else if (EntityType.TENANT.equals(ownerId.getEntityType())) { PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByFilter(DeviceInfoFilter.builder().tenantId((TenantId) ownerId).customerId(new CustomerId(CustomerId.NULL_UUID)).build(), pageLink), 1000); - deviceIdInfos.forEach(deviceInfo -> ownerEntities.add(deviceInfo.getId())); + deviceIdInfos.forEach(deviceInfo -> ownedEntities.add(deviceInfo.getId())); + PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantIdAndCustomerId((TenantId) ownerId, new CustomerId(CustomerId.NULL_UUID), pageLink), 1000); - assets.forEach(asset -> ownerEntities.add(asset.getId())); + assets.forEach(asset -> ownedEntities.add(asset.getId())); + + PageDataIterable customers = new PageDataIterable<>(pageLink -> customerService.findCustomersByTenantId((TenantId) ownerId, pageLink), 1000); + customers.forEach(customer -> ownedEntities.add(customer.getId())); } - return ownerEntities; + return ownedEntities; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index aa308919db..b9fd38f1e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.JobManager; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; @@ -148,6 +149,9 @@ public class EntityStateSourcingListener { case JOB -> { onJobUpdate((Job) event.getEntity()); } + case CUSTOMER -> { + tbClusterService.onCustomerUpdated((Customer) event.getEntity(), (Customer) event.getOldEntity()); + } default -> { } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 460bca52bb..63f8a9bf2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.Optional; +import java.util.Set; @TbCoreComponent @Service @@ -53,7 +53,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); checkForEntityChange(existingCf, calculatedField); } - checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkEntity(tenantId, calculatedField.getEntityId(), calculatedField.getType()); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); return savedCalculatedField; @@ -70,7 +70,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp @Override public PageData findByTenantIdAndEntityId(TenantId tenantId, EntityId entityId, CalculatedFieldType type, PageLink pageLink) { - checkEntityExistence(tenantId, entityId); + checkEntity(tenantId, entityId, type); return calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId, type, pageLink); } @@ -95,12 +95,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { - if (CalculatedField.SUPPORTED_ENTITIES.contains(entityId.getEntityType())) { - Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) - .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); - } else { - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + private void checkEntity(TenantId tenantId, EntityId entityId, CalculatedFieldType type) { + EntityType entityType = entityId.getEntityType(); + Set supportedTypes = CalculatedField.SUPPORTED_ENTITIES.get(entityType); + if (supportedTypes == null || supportedTypes.isEmpty()) { + throw new IllegalArgumentException("Entity type '" + entityType + "' does not support calculated fields"); + } else if (type != null && !supportedTypes.contains(type)) { + throw new IllegalArgumentException("Entity type '" + entityType + "' does not support '" + type + "' calculated fields"); + } else if (entityService.fetchEntity(tenantId, entityId).isEmpty()) { + throw new IllegalArgumentException(entityType.getNormalName() + " with id [" + entityId.getId() + "] does not exist."); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 9ad9021072..f7f12ac30e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -468,6 +469,17 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(resource.getTenantId(), resource.getId(), ComponentLifecycleEvent.DELETED); } + @Override + public void onCustomerUpdated(Customer customer, Customer oldCustomer) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(customer.getTenantId()) + .entityId(customer.getId()) + .event(oldCustomer == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED) + .ownerChanged(false) // for compatibility with PE + .build(); + broadcast(msg); + } + private void broadcastEntityChangeToTransport(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName(); log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 402db223a1..7382ef1c4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -209,9 +209,14 @@ public abstract class AbstractConsumerService { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(originatorId, "{\"temperature\":100}"); + postTelemetry(deviceId, "{\"temperature\":100}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isSeverityUpdated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(originatorId, "{\"temperature\":101}"); + postTelemetry(deviceId, "{\"temperature\":101}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isUpdated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); }); - postTelemetry(originatorId, "{\"temperature\":20}"); + postTelemetry(deviceId, "{\"temperature\":20}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCleared()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -164,10 +167,10 @@ public class AlarmRulesTest extends AbstractControllerTest { AlarmSeverity.CRITICAL, new Condition(simpleExpression, null, null) ); - CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, null); - postTelemetry(originatorId, "{\"temperature\":100}"); + postTelemetry(deviceId, "{\"temperature\":100}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -194,14 +197,14 @@ public class AlarmRulesTest extends AbstractControllerTest { AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", eventsCountCritical, null) ); - CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, null); for (int i = 0; i < 4; i++) { - postTelemetry(originatorId, "{\"temperature\":50}"); + postTelemetry(deviceId, "{\"temperature\":50}"); Thread.sleep(10); } assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); - postTelemetry(originatorId, "{\"temperature\":50}"); + postTelemetry(deviceId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); @@ -210,7 +213,7 @@ public class AlarmRulesTest extends AbstractControllerTest { }); for (int i = 0; i < 5; i++) { - postTelemetry(originatorId, "{\"temperature\":50}"); + postTelemetry(deviceId, "{\"temperature\":50}"); Thread.sleep(10); } checkAlarmResult(calculatedField, alarmResult -> { @@ -236,9 +239,9 @@ public class AlarmRulesTest extends AbstractControllerTest { ); Condition clearRule = new Condition("return powerConsumption < 3000;", null, createDurationMs); - CalculatedField calculatedField = createAlarmCf(originatorId, "High power consumption during 5 seconds", + CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", arguments, createRules, clearRule); - postTelemetry(originatorId, "{\"powerConsumption\":3500}"); + postTelemetry(deviceId, "{\"powerConsumption\":3500}"); Thread.sleep(createDurationMs - 2000); assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); @@ -272,11 +275,11 @@ public class AlarmRulesTest extends AbstractControllerTest { device.setCustomerId(customerId); device = doPost("/api/device", device, Device.class); - CalculatedField calculatedField = createAlarmCf(originatorId, "High Temperature Alarm", + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, null); postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"temperatureThreshold\":50}"); - postTelemetry(originatorId, "{\"temperature\":51}"); + postTelemetry(deviceId, "{\"temperature\":51}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -289,20 +292,17 @@ public class AlarmRulesTest extends AbstractControllerTest { Argument locationArgument = new Argument(); locationArgument.setRefEntityKey(new ReferencedEntityKey("location", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); locationArgument.setDefaultValue("unknown"); + originatorId = customerId; Argument locationFilterArgument = new Argument(); locationFilterArgument.setRefEntityKey(new ReferencedEntityKey("locationFilter", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); locationFilterArgument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); locationFilterArgument.setDefaultValue("None"); - loginSysAdmin(); - postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"locationFilter\":\"Kyiv\"}"); - loginTenantAdmin(); Map arguments = Map.of( "location", locationArgument, "locationFilter", locationFilterArgument ); - originatorId = customerId; Map createRules = Map.of( AlarmSeverity.INDETERMINATE, new Condition(createSimpleExpression( @@ -316,6 +316,9 @@ public class AlarmRulesTest extends AbstractControllerTest { CalculatedField calculatedField = createAlarmCf(customerId, "New resident", arguments, createRules, clearRule); + loginSysAdmin(); + postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"locationFilter\":\"Kyiv\"}"); + loginTenantAdmin(); postAttributes(customerId, AttributeScope.SERVER_SCOPE, "{\"location\":\"Ukraine, Kyiv\"}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isCreated()).isTrue(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 1805788007..701c22587b 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.cluster; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; @@ -130,6 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCustomerUpdated(Customer customer, Customer oldCustomer); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index f39ce27169..0d2543d4c2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.validation.NoXss; import java.io.Serial; import java.util.Collections; import java.util.EnumSet; +import java.util.Map; import java.util.Set; @Schema @@ -52,9 +53,14 @@ public class CalculatedField extends BaseData implements HasN @Serial private static final long serialVersionUID = 4491966747773381420L; - public static final Set SUPPORTED_ENTITIES = Collections.unmodifiableSet(EnumSet.of( - EntityType.DEVICE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE, EntityType.CUSTOMER - )); + public static final Map> SUPPORTED_ENTITIES = Map.of( + EntityType.DEVICE, CalculatedFieldType.all, + EntityType.ASSET, CalculatedFieldType.all, + EntityType.DEVICE_PROFILE, CalculatedFieldType.all, + EntityType.ASSET_PROFILE, CalculatedFieldType.all, + EntityType.CUSTOMER, Set.of(CalculatedFieldType.ALARM) + ); + public static final Set SUPPORTED_REFERENCED_ENTITIES = Collections.unmodifiableSet(EnumSet.of( EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT )); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 3399808a35..7052ab70e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -15,9 +15,17 @@ */ package org.thingsboard.server.common.data.cf; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + public enum CalculatedFieldType { + SIMPLE, SCRIPT, GEOFENCING, - ALARM + ALARM; + + public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index b06902d95e..78237c76c8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -144,9 +144,10 @@ public class CustomerServiceImpl extends AbstractCachedEntityService Date: Mon, 6 Oct 2025 18:08:23 +0300 Subject: [PATCH 023/122] Added PropagationArgumentEntryTest & PropagationCalculatedFieldStateTest & integration tests for propagation CF --- .../PropagationCalculatedFieldState.java | 9 +- .../cf/CalculatedFieldIntegrationTest.java | 166 ++++++++++++ .../state/PropagationArgumentEntryTest.java | 143 ++++++++++ .../PropagationCalculatedFieldStateTest.java | 246 ++++++++++++++++++ .../utils/CalculatedFieldUtilsTest.java | 5 +- 5 files changed, 562 insertions(+), 7 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java index d039d7aa7b..23c4af5bce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -43,9 +43,12 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState } @Override - public void init(CalculatedFieldCtx ctx) { - super.init(ctx); - requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT); + public boolean isReady() { + if (!super.isReady()) { + return false; + } + ArgumentEntry propagationArg = arguments.get(PROPAGATION_CONFIG_ARGUMENT); + return propagationArg != null && !propagationArg.isEmpty(); } @Override diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 5d95a572b8..92e4438ff3 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; @@ -24,6 +25,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -34,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; @@ -944,6 +947,169 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); } + @Test + public void testPropagationCalculatedField_withExpression() throws Exception { + // --- Arrange entities --- + Device device = createDevice("Propagation Device With Expression", "sn-prop-1"); + Asset asset1 = createAsset("Propagated Asset 1", null); + Asset asset2 = createAsset("Propagated Asset 2", null); + + // Create relations FROM assets TO device + EntityRelation rel1 = new EntityRelation(asset1.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + EntityRelation rel2 = new EntityRelation(asset2.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + doPost("/api/relation", rel1).andExpect(status().isOk()); + doPost("/api/relation", rel2).andExpect(status().isOk()); + + // Telemetry on device + doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope", + JacksonUtil.toJsonNode("{\"temperature\":12.5}")).andExpect(status().isOk()); + + // --- Build CF: PROPAGATION with expression --- + CalculatedField cf = new CalculatedField(); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.PROPAGATION); + cf.setName("Propagation CF (expr)"); + cf.setConfigurationVersion(1); + + PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setDirection(EntitySearchDirection.TO); + cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setApplyExpressionToResolvedArguments(true); + + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("t", arg)); + + cfg.setExpression("{\"testResult\": t * 2}"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + cfg.setOutput(output); + + cf.setConfiguration(cfg); + + doPost("/api/calculatedField", cf, CalculatedField.class); + + // --- Assert propagated calculation (expression applied) --- + await().alias("propagation expr mode evaluation") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode attrs1 = getServerAttributes(asset1.getId(), "testResult"); + ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult"); + assertThat(attrs1).isNotNull(); + assertThat(attrs2).isNotNull(); + assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(25.0); + assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(25.0); + }); + + String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + asset1.getId().getId(), EntityType.ASSET, + EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE + ); + doDelete(deleteUrl).andExpect(status().isOk()); + doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/SERVER_SCOPE?keys=testResult").andExpect(status().isOk()); + + doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope", + JacksonUtil.toJsonNode("{\"temperature\":25}")).andExpect(status().isOk()); + + // --- Assert propagated calculation (expression applied with new temperature argument and one relation removed) --- + await().alias("propagation expr mode evaluation after temperature update") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode attrs1 = getServerAttributes(asset1.getId(), "testResult"); + ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult"); + assertThat(attrs1).isNullOrEmpty(); + assertThat(attrs2).isNotNull(); + assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(50); + }); + } + + @Test + public void testPropagationCalculatedField_withoutExpression() throws Exception { + // --- Arrange entities --- + Device device = createDevice("Propagation Device Without Expression", "sn-prop-2"); + Asset asset1 = createAsset("Propagated Asset 1", null); + Asset asset2 = createAsset("Propagated Asset 2", null); + + // Create relations FROM assets TO device + EntityRelation rel1 = new EntityRelation(asset1.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + EntityRelation rel2 = new EntityRelation(asset2.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + doPost("/api/relation", rel1).andExpect(status().isOk()); + doPost("/api/relation", rel2).andExpect(status().isOk()); + + // Telemetry on device + long ts = System.currentTimeMillis() - 300000L; + postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5}}", ts)); + + // --- Build CF: PROPAGATION without expression --- + CalculatedField cf = new CalculatedField(); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.PROPAGATION); + cf.setName("Propagation CF (args-only)"); + cf.setConfigurationVersion(1); + + PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setDirection(EntitySearchDirection.TO); + cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode + + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("t", arg)); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + cfg.setOutput(output); + + cf.setConfiguration(cfg); + + doPost("/api/calculatedField", cf, CalculatedField.class); + + // --- Assert propagated calculation (arguments-only mode) --- + await().alias("propagation args-only evaluation") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperature"); + ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperature"); + assertThat(telemetry1).isNotNull(); + assertThat(telemetry2).isNotNull(); + assertThat(telemetry1.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(telemetry1.get("temperature").get(0).get("value").asDouble()).isEqualTo(12.5); + assertThat(telemetry2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(telemetry2.get("temperature").get(0).get("value").asDouble()).isEqualTo(12.5); + }); + + String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + asset1.getId().getId(), EntityType.ASSET, + EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE + ); + doDelete(deleteUrl).andExpect(status().isOk()); + doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperature&deleteAllDataForKeys=true").andExpect(status().isOk()); + + // Update telemetry on device + long newTs = System.currentTimeMillis() - 300000L; + postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs)); + + // --- Assert propagated calculation (arguments-only mode after update) --- + await().alias("propagation args-only evaluation after temperature update") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperature"); + ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperature"); + assertThat(telemetry1).isNotNull(); + assertThat(telemetry2).isNotNull(); + assertThat(telemetry1.get("temperature").get(0).get("value")).isEqualTo(NullNode.instance); + assertThat(telemetry2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); + assertThat(telemetry2.get("temperature").get(0).get("value").asDouble()).isEqualTo(25); + }); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java new file mode 100644 index 0000000000..9c8a788e15 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfPropagationArg; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class PropagationArgumentEntryTest { + + private final AssetId ENTITY_1_ID = new AssetId(UUID.fromString("b0a8637d-6d67-43d5-a483-c0e391afe805")); + private final AssetId ENTITY_2_ID = new AssetId(UUID.fromString("7bd85073-ded5-414f-a2ef-bd56ad3dbf6a")); + private final AssetId ENTITY_3_ID = new AssetId(UUID.fromString("d64f3e51-2ec2-472f-b475-b095ef8bdc70")); + + private PropagationArgumentEntry entry; + + @BeforeEach + void setUp() { + List propagationEntityIds = new ArrayList<>(); + propagationEntityIds.add(ENTITY_1_ID); + propagationEntityIds.add(ENTITY_2_ID); + entry = new PropagationArgumentEntry(propagationEntityIds); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.PROPAGATION); + } + + @Test + void testIsEmpty() { + PropagationArgumentEntry emptyEntry = new PropagationArgumentEntry(List.of()); + assertThat(emptyEntry.isEmpty()).isTrue(); + } + + @Test + void testIsEmptyWhenNullList() { + PropagationArgumentEntry nullListEntry = new PropagationArgumentEntry(null); + assertThat(nullListEntry.isEmpty()).isTrue(); + } + + @Test + void testGetValueReturnsPropagationIds() { + assertThat(entry.getValue()).isInstanceOf(List.class); + @SuppressWarnings("unchecked") + List value = (List) entry.getValue(); + assertThat(value).containsExactly(ENTITY_1_ID, ENTITY_2_ID); + } + + @Test + void testUpdateEntryWhenSingleEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new SingleValueArgumentEntry())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for propagation argument entry: SINGLE_VALUE"); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for propagation argument entry: TS_ROLLING"); + } + + @Test + void testUpdateEntryReplacesWithNewIds() { + var newIds = new ArrayList(List.of(ENTITY_3_ID, ENTITY_1_ID)); + var updated = new PropagationArgumentEntry(newIds); + + boolean changed = entry.updateEntry(updated); + + assertThat(changed).isTrue(); + assertThat(entry.getPropagationEntityIds()).containsExactlyElementsOf(newIds); + } + + @Test + void testUpdateEntryClearsWhenNewEntryIsEmpty() { + var updatedEmpty = new PropagationArgumentEntry(List.of()); + + boolean changed = entry.updateEntry(updatedEmpty); + + assertThat(changed).isTrue(); + assertThat(entry.getPropagationEntityIds()).isEmpty(); + } + + @Test + void testUpdateEntryClearsWhenNewEntryIsNullList() { + var updatedNull = new PropagationArgumentEntry(null); + + boolean changed = entry.updateEntry(updatedNull); + + assertThat(changed).isTrue(); + assertThat(entry.getPropagationEntityIds()).isEmpty(); + } + + @Test + @SuppressWarnings("unchecked") + void testToTbelCfArgWithValues() { + TbelCfArg arg = entry.toTbelCfArg(); + assertThat(arg).isInstanceOf(TbelCfPropagationArg.class); + + TbelCfPropagationArg tbelCfPropagationArg = (TbelCfPropagationArg) arg; + assertThat(tbelCfPropagationArg.getValue()).isInstanceOf(List.class); + assertThat((List) tbelCfPropagationArg.getValue()).containsExactly(ENTITY_1_ID, ENTITY_2_ID); + } + + + @Test + @SuppressWarnings("unchecked") + void testToTbelCfArgWithEmptyValues() { + var empty = new PropagationArgumentEntry(List.of()); + TbelCfArg emptyArg = empty.toTbelCfArg(); + assertThat(emptyArg).isInstanceOf(TbelCfPropagationArg.class); + + TbelCfPropagationArg tbelCfPropagationArg = (TbelCfPropagationArg) emptyArg; + assertThat(tbelCfPropagationArg.getValue()).isInstanceOf(List.class); + assertThat((List) tbelCfPropagationArg.getValue()).isEmpty(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java new file mode 100644 index 0000000000..234a0bdb06 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java @@ -0,0 +1,246 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.stats.DefaultStatsFactory; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; + +@SpringBootTest(classes = {SimpleMeterRegistry.class, DefaultStatsFactory.class, DefaultTbelInvokeService.class}) +public class PropagationCalculatedFieldStateTest { + + private static final String TEMPERATURE_ARGUMENT_NAME = "t"; + private static final String TEST_RESULT_EXPRESSION_KEY = "testResult"; + private static final double TEMPERATURE_VALUE = 12.5; + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("6c3513cb-85e7-4510-8746-1ba01859a8ce")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("be960a50-c029-4698-b2ec-c56a543c561c")); + private final AssetId ASSET_ID_1 = new AssetId(UUID.fromString("d26f0e5b-7d7d-4a61-9f5e-08ab97b30734")); + private final AssetId ASSET_ID_2 = new AssetId(UUID.fromString("1933a317-4df5-4d36-9800-68aded74579b")); + + private final SingleValueArgumentEntry singleValueArgEntry = + new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", TEMPERATURE_VALUE), 99L); + + private final PropagationArgumentEntry propagationArgEntry = + new PropagationArgumentEntry(new ArrayList<>(List.of(ASSET_ID_2, ASSET_ID_1))); + + private PropagationCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @MockitoBean + private ApiLimitService apiLimitService; + + @MockitoBean + private ActorSystemContext actorSystemContext; + + @BeforeEach + void setUp() { + when(actorSystemContext.getTbelInvokeService()).thenReturn(tbelInvokeService); + when(actorSystemContext.getApiLimitService()).thenReturn(apiLimitService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + } + + void initCtxAndState(boolean applyExpressionToResolvedArguments) { + ctx = new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext); + ctx.init(); + + state = new PropagationCalculatedFieldState(ctx.getEntityId()); + state.init(ctx); + } + + @Test + void testType() { + initCtxAndState(false); + assertThat(state.getType()).isEqualTo(CalculatedFieldType.PROPAGATION); + } + + @Test + void testInitAddsRequiredArgument() { + initCtxAndState(false); + assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME); + } + + @Test + void testIsReadyReturnFalseWhenNoArgumentsSet() { + initCtxAndState(false); + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenPropagationArgIsNull() { + initCtxAndState(false); + state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenPropagationArgIsEmpty() { + initCtxAndState(false); + state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())); + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenPropagationArgHasEntities() { + initCtxAndState(false); + state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); + assertThat(state.isReady()).isTrue(); + } + + + @Test + void testPerformCalculationWithEmptyPropagationArg() throws Exception { + initCtxAndState(false); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())); + + PropagationCalculatedFieldResult result = performCalculation(); + + assertThat(result).isNotNull(); + assertThat(result.isEmpty()).isTrue(); + assertThat(result.getPropagationEntityIds()).isNullOrEmpty(); + } + + @Test + void testPerformCalculationWithArgumentsOnlyMode() throws Exception { + initCtxAndState(false); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); + state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); + + PropagationCalculatedFieldResult propagationResult = performCalculation(); + + assertThat(propagationResult).isNotNull(); + assertThat(propagationResult.isEmpty()).isFalse(); + assertThat(propagationResult.getPropagationEntityIds()).containsExactly(ASSET_ID_2, ASSET_ID_1); + + TelemetryCalculatedFieldResult result = propagationResult.getResult(); + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(OutputType.ATTRIBUTES); + assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); + + ObjectNode expectedNode = JacksonUtil.newObjectNode(); + JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue()); + + assertThat(result.getResult()).isEqualTo(expectedNode); + } + + @Test + void testPerformCalculationWithExpressionResultMode() throws Exception { + initCtxAndState(true); + state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); + state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); + + PropagationCalculatedFieldResult propagationResult = performCalculation(); + + assertThat(propagationResult).isNotNull(); + assertThat(propagationResult.isEmpty()).isFalse(); + assertThat(propagationResult.getPropagationEntityIds()).containsExactly(ASSET_ID_2, ASSET_ID_1); + + TelemetryCalculatedFieldResult result = propagationResult.getResult(); + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(OutputType.ATTRIBUTES); + assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); + + ObjectNode expectedNode = JacksonUtil.newObjectNode(); + expectedNode.put(TEST_RESULT_EXPRESSION_KEY, TEMPERATURE_VALUE * 2); + + assertThat(result.getResult()).isEqualTo(expectedNode); + } + + private CalculatedField getCalculatedField(boolean applyExpressionToResolvedArguments) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.PROPAGATION); + calculatedField.setName("Test Propagation CF"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(applyExpressionToResolvedArguments)); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(boolean applyExpressionToResolvedArguments) { + var config = new PropagationCalculatedFieldConfiguration(); + + config.setDirection(EntitySearchDirection.TO); + config.setRelationType(EntityRelation.CONTAINS_TYPE); + config.setApplyExpressionToResolvedArguments(applyExpressionToResolvedArguments); + + Argument temperatureArg = new Argument(); + ReferencedEntityKey tempKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + temperatureArg.setRefEntityKey(tempKey); + + config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg)); + config.setExpression("{" + TEST_RESULT_EXPRESSION_KEY + ": " + TEMPERATURE_ARGUMENT_NAME + " * 2}"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + config.setOutput(output); + + return config; + } + + private PropagationCalculatedFieldResult performCalculation() throws ExecutionException, InterruptedException { + return (PropagationCalculatedFieldResult) state.performCalculation(Collections.emptyMap(), ctx).get(); + } +} diff --git a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java index 0d57f90206..505837ece2 100644 --- a/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java @@ -50,7 +50,6 @@ import static org.mockito.Mockito.mock; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; - @ExtendWith(MockitoExtension.class) class CalculatedFieldUtilsTest { @@ -94,11 +93,9 @@ class CalculatedFieldUtilsTest { CalculatedFieldState state = new GeofencingCalculatedFieldState(DEVICE_ID); state.update(Map.of("geofencingArgumentTest", geofencingArgumentEntry), mock(CalculatedFieldCtx.class)); - // when CalculatedFieldStateProto proto = toProto(stateId, state); - - // then CalculatedFieldState fromProto = CalculatedFieldUtils.fromProto(stateId, proto); + assertThat(fromProto) .usingRecursiveComparison() .ignoringFields("requiredArguments") From 56414f1a101a917183456a81357db5b82a828e0d Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Tue, 7 Oct 2025 10:44:43 +0300 Subject: [PATCH 024/122] Broadcast customer update msg to cores --- .../server/service/queue/DefaultTbClusterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f7f12ac30e..f1494658a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -609,7 +609,8 @@ public class DefaultTbClusterService implements TbClusterService { EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE, EntityType.JOB, - EntityType.TB_RESOURCE) + EntityType.TB_RESOURCE, + EntityType.CUSTOMER) || (entityType == EntityType.ASSET && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || (entityType == EntityType.DEVICE && msg.getEvent() == ComponentLifecycleEvent.UPDATED) ) { From 265b63dc06d6bd8fad15a0e152c621bc174be245 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Tue, 7 Oct 2025 13:01:11 +0300 Subject: [PATCH 025/122] fixed NPE on reference entities check + added basic controller tests for Propagation CF --- .../CalculatedFieldControllerTest.java | 95 +++++++++++++++++-- .../AlarmCalculatedFieldConfiguration.java | 2 - ...entsBasedCalculatedFieldConfiguration.java | 7 ++ .../CalculatedFieldConfiguration.java | 3 +- ...eofencingCalculatedFieldConfiguration.java | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 3 +- 6 files changed, 96 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 27622b347a..4da8da8564 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; @@ -35,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoor import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.security.Authority; @@ -44,6 +46,7 @@ import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingReportStrategy.REPORT_TRANSITION_EVENTS_AND_PRESENCE_STATUS; @@ -81,7 +84,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testSaveCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + CalculatedField calculatedField = getSimpleCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -109,7 +112,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testSaveGeofencingCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - CalculatedField calculatedField = getCalculatedField(testDevice.getId(), getGeofencingCalculatedFieldConfig()); + CalculatedField calculatedField = getCalculatedField(testDevice.getId(), CalculatedFieldType.GEOFENCING); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -134,10 +137,48 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { .andExpect(status().isOk()); } + @Test + public void testSavePropagationCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId(), CalculatedFieldType.PROPAGATION); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getPropagationCalculatedFieldConfig()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testSavePropagationCalculatedFieldWithNullArguments() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId(), CalculatedFieldType.PROPAGATION, getPropagationCalculatedFieldConfig(null)); + + doPost("/api/calculatedField", calculatedField) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("arguments must not be empty"))); + } + @Test public void testGetCalculatedFieldById() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + CalculatedField calculatedField = getSimpleCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); @@ -152,7 +193,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testDeleteCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + CalculatedField calculatedField = getSimpleCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -163,17 +204,27 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); } - private CalculatedField getCalculatedField(DeviceId deviceId) { - return getCalculatedField(deviceId, getSimpleCalculatedFieldConfig()); + private CalculatedField getSimpleCalculatedField(DeviceId deviceId) { + return getCalculatedField(deviceId, CalculatedFieldType.SIMPLE); + } + + private CalculatedField getCalculatedField(DeviceId deviceId, CalculatedFieldType cfType) { + return getCalculatedField(deviceId, cfType, null); } - private CalculatedField getCalculatedField(DeviceId deviceId, CalculatedFieldConfiguration configuration) { + private CalculatedField getCalculatedField(DeviceId deviceId, CalculatedFieldType cfType, CalculatedFieldConfiguration customConfiguration) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setType(cfType); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(configuration); + if (customConfiguration != null) { + calculatedField.setConfiguration(customConfiguration); + } else switch (cfType) { + case SIMPLE -> calculatedField.setConfiguration(getSimpleCalculatedFieldConfig()); + case GEOFENCING -> calculatedField.setConfiguration(getGeofencingCalculatedFieldConfig()); + case PROPAGATION -> calculatedField.setConfiguration(getPropagationCalculatedFieldConfig()); + } calculatedField.setVersion(1L); return calculatedField; } @@ -198,6 +249,32 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { return config; } + private CalculatedFieldConfiguration getPropagationCalculatedFieldConfig() { + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + return getPropagationCalculatedFieldConfig(Map.of("t", arg)); + } + + private CalculatedFieldConfiguration getPropagationCalculatedFieldConfig(Map arguments) { + var config = new PropagationCalculatedFieldConfiguration(); + + config.setRelationType(EntityRelation.CONTAINS_TYPE); + config.setDirection(EntitySearchDirection.TO); + + config.setApplyExpressionToResolvedArguments(false); + config.setExpression(null); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + config.setArguments(arguments); + + return config; + } + private CalculatedFieldConfiguration getSimpleCalculatedFieldConfig() { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java index c2925d5ed6..d1e0a33916 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -29,8 +29,6 @@ import java.util.Map; @Data public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { - @Valid - @NotEmpty private Map arguments; @Valid diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java index 31c95b2119..f422869c95 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentsBasedCalculatedFieldConfiguration.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; @@ -24,9 +26,14 @@ import java.util.stream.Collectors; public interface ArgumentsBasedCalculatedFieldConfiguration extends CalculatedFieldConfiguration { + @Valid + @NotEmpty Map getArguments(); default List getReferencedEntities() { + if (getArguments() == null) { + return List.of(); + } return getArguments().values().stream() .map(Argument::getRefEntityId) .filter(Objects::nonNull) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 8676c6060f..bdf2bdcb93 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -55,7 +54,7 @@ public interface CalculatedFieldConfiguration { @JsonIgnore default List getReferencedEntities() { - return Collections.emptyList(); + return List.of(); } default CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java index ec97ad6116..786d740a26 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java @@ -56,7 +56,7 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal @Override public List getReferencedEntities() { - return zoneGroups.values().stream().map(ZoneGroupConfiguration::getRefEntityId).filter(Objects::nonNull).toList(); + return zoneGroups == null ? List.of() : zoneGroups.values().stream().map(ZoneGroupConfiguration::getRefEntityId).filter(Objects::nonNull).toList(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index c0cb886747..7a9fccf401 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -57,8 +57,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField) { - CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); - return doSave(calculatedField, oldCalculatedField); + return save(calculatedField, true); } @Override From 2789acd2ddde1b405ff04cc891951e92f25916ea Mon Sep 17 00:00:00 2001 From: dshvaika Date: Tue, 7 Oct 2025 17:12:54 +0300 Subject: [PATCH 026/122] configuration updates + tests --- ...opagationCalculatedFieldConfiguration.java | 14 +-- ...SupportedCalculatedFieldConfiguration.java | 3 + .../geofencing/EntityCoordinates.java | 13 +- ...eofencingCalculatedFieldConfiguration.java | 14 +-- .../geofencing/ZoneGroupConfiguration.java | 10 +- ...ationCalculatedFieldConfigurationTest.java | 116 ++++++++++++++++++ ...ortedCalculatedFieldConfigurationTest.java | 2 +- .../geofencing/EntityCoordinatesTest.java | 31 ----- ...ncingCalculatedFieldConfigurationTest.java | 29 +---- .../ZoneGroupConfigurationTest.java | 18 --- 10 files changed, 142 insertions(+), 108 deletions(-) create mode 100644 common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index b592264d6c..3394813d3f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.StringUtils; @@ -30,7 +32,9 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField public static final String PROPAGATION_CONFIG_ARGUMENT = "propagationCtx"; + @NotNull private EntitySearchDirection direction; + @NotBlank private String relationType; private boolean applyExpressionToResolvedArguments; @@ -44,20 +48,14 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField public void validate() { baseCalculatedFieldRestriction(); propagationRestriction(); - if (direction == null) { - throw new IllegalArgumentException("Propagation calculated field direction must be specified!"); - } - if (StringUtils.isBlank(relationType)) { - throw new IllegalArgumentException("Propagation calculated field relation type must be specified!"); - } if (!applyExpressionToResolvedArguments) { arguments.forEach((name, argument) -> { if (argument.getRefEntityKey() == null) { throw new IllegalArgumentException("Argument: '" + name + "' doesn't have reference entity key configured!"); } if (argument.getRefEntityKey().getType() == ArgumentType.TS_ROLLING) { - throw new IllegalArgumentException("Argument type: 'Time series rolling' detected for argument: '" + name + "'! " + - "Only 'Attribute' or 'Latest telemetry' arguments are allowed for in 'Arguments only' propagation mode!"); + throw new IllegalArgumentException("Argument type: 'Time series rolling' detected for argument: '" + name + "'. " + + "Only 'Attribute' or 'Latest telemetry' arguments are allowed for 'Arguments only' propagation mode!"); } }); } else if (StringUtils.isBlank(expression)) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java index d0c5786f62..e1e8ca1a9b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import jakarta.validation.constraints.PositiveOrZero; + public interface ScheduledUpdateSupportedCalculatedFieldConfiguration extends CalculatedFieldConfiguration { boolean isScheduledUpdateEnabled(); + @PositiveOrZero int getScheduledUpdateInterval(); void setScheduledUpdateInterval(int interval); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinates.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinates.java index 9ea5c19e8c..ad31293061 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinates.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinates.java @@ -16,8 +16,8 @@ package org.thingsboard.server.common.data.cf.configuration.geofencing; +import jakarta.validation.constraints.NotBlank; import lombok.Data; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -30,18 +30,11 @@ public class EntityCoordinates { public static final String ENTITY_ID_LATITUDE_ARGUMENT_KEY = "latitude"; public static final String ENTITY_ID_LONGITUDE_ARGUMENT_KEY = "longitude"; + @NotBlank private final String latitudeKeyName; + @NotBlank private final String longitudeKeyName; - public void validate() { - if (StringUtils.isBlank(latitudeKeyName)) { - throw new IllegalArgumentException("Entity coordinates latitude key name must be specified!"); - } - if (StringUtils.isBlank(longitudeKeyName)) { - throw new IllegalArgumentException("Entity coordinates longitude key name must be specified!"); - } - } - public Map toArguments() { return Map.of( ENTITY_ID_LATITUDE_ARGUMENT_KEY, toArgument(latitudeKeyName), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java index 786d740a26..47d344fe1b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java @@ -16,6 +16,8 @@ package org.thingsboard.server.common.data.cf.configuration.geofencing; import com.fasterxml.jackson.annotation.JsonIgnore; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -32,7 +34,12 @@ import java.util.Objects; @Data public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration, ScheduledUpdateSupportedCalculatedFieldConfiguration { + @Valid + @NotNull private EntityCoordinates entityCoordinates; + + @Valid + @NotNull private Map zoneGroups; private boolean scheduledUpdateEnabled; @@ -66,13 +73,6 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal @Override public void validate() { - if (entityCoordinates == null) { - throw new IllegalArgumentException("Geofencing calculated field entity coordinates must be specified!"); - } - entityCoordinates.validate(); - if (zoneGroups == null || zoneGroups.isEmpty()) { - throw new IllegalArgumentException("Geofencing calculated field must contain at least one geofencing zone group defined!"); - } zoneGroups.forEach((key, value) -> value.validate(key)); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java index 775f711a5e..a06cb242cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfiguration.java @@ -17,6 +17,8 @@ package org.thingsboard.server.common.data.cf.configuration.geofencing; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.springframework.lang.Nullable; import org.thingsboard.server.common.data.AttributeScope; @@ -36,8 +38,10 @@ public class ZoneGroupConfiguration { private EntityId refEntityId; private CfArgumentDynamicSourceConfiguration refDynamicSourceConfiguration; + @NotBlank private final String perimeterKeyName; + @NotNull private final GeofencingReportStrategy reportStrategy; private final boolean createRelationsWithMatchedZones; @@ -48,12 +52,6 @@ public class ZoneGroupConfiguration { if (EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY.equals(name) || EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY.equals(name)) { throw new IllegalArgumentException("Name '" + name + "' is reserved and cannot be used for zone group!"); } - if (StringUtils.isBlank(perimeterKeyName)) { - throw new IllegalArgumentException("Perimeter key name must be specified for '" + name + "' zone group!"); - } - if (reportStrategy == null) { - throw new IllegalArgumentException("Report strategy must be specified for '" + name + "' zone group!"); - } if (refDynamicSourceConfiguration != null) { refDynamicSourceConfiguration.validate(); } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java new file mode 100644 index 0000000000..eb0591e32b --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; + +@ExtendWith(MockitoExtension.class) +public class PropagationCalculatedFieldConfigurationTest { + + @Test + void typeShouldBePropagation() { + var cfg = new PropagationCalculatedFieldConfiguration(); + assertThat(cfg.getType()).isEqualTo(CalculatedFieldType.PROPAGATION); + } + + @Test + void validateShouldThrowWhenUsedReservedPropagationArgumentName() { + var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setArguments(Map.of(PROPAGATION_CONFIG_ARGUMENT, new Argument())); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument name '" + PROPAGATION_CONFIG_ARGUMENT + "' is reserved and cannot be used."); + } + + @Test + void validateShouldThrowWhenUsedReservedCtxArgumentName() { + var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setArguments(Map.of("ctx", new Argument())); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument name 'ctx' is reserved and cannot be used."); + } + + @Test + void validateShouldThrowWhenReferencedEntityKeyIsNotSet() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argument = new Argument(); + cfg.setArguments(Map.of("someArgumentName", argument)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument: 'someArgumentName' doesn't have reference entity key configured!"); + } + + @Test + void validateShouldThrowWhenReferencedEntityKeyTypeIsTsRolling() { + var cfg = new PropagationCalculatedFieldConfiguration(); + ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey("someKey", ArgumentType.TS_ROLLING, null); + Argument argument = new Argument(); + argument.setRefEntityKey(referencedEntityKey); + cfg.setArguments(Map.of("someArgumentName", argument)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument type: 'Time series rolling' detected for argument: 'someArgumentName'. " + + "Only 'Attribute' or 'Latest telemetry' arguments are allowed for 'Arguments only' propagation mode!"); + } + + @Test + void validateShouldThrowWhenExpressionIsNotSet() { + var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setArguments(Map.of("someArgumentName", new Argument())); + cfg.setApplyExpressionToResolvedArguments(true); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Expression must be specified for 'Expression result' propagation mode!"); + } + + @Test + void validateToPropagationArgumentMethodCallReturnCorrectArgument() { + var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setDirection(EntitySearchDirection.TO); + cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + + Argument propagationArgument = cfg.toPropagationArgument(); + assertThat(propagationArgument).isNotNull(); + assertThat(propagationArgument.getRefEntityId()).isNull(); + assertThat(propagationArgument.getRefEntityKey()).isNull(); + assertThat(propagationArgument.getDefaultValue()).isNull(); + assertThat(propagationArgument.getTimeWindow()).isNull(); + assertThat(propagationArgument.getLimit()).isNull(); + + assertThat(propagationArgument.getRefDynamicSourceConfiguration()) + .isNotNull() + .isInstanceOf(RelationPathQueryDynamicSourceConfiguration.class); + var refDynamicSourceConfiguration = (RelationPathQueryDynamicSourceConfiguration) propagationArgument.getRefDynamicSourceConfiguration(); + assertThat(refDynamicSourceConfiguration.getLevels()).isNotEmpty().hasSize(1); + + var relationPathLevel = refDynamicSourceConfiguration.getLevels().get(0); + assertThat(relationPathLevel.direction()).isEqualTo(EntitySearchDirection.TO); + assertThat(relationPathLevel.relationType()).isEqualTo(EntityRelation.CONTAINS_TYPE); + } + +} diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java index 3c0956bd08..15a191be97 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java @@ -29,7 +29,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; public class ScheduledUpdateSupportedCalculatedFieldConfigurationTest { @Test - void validateShouldThrowWhenScheduledUpdateIntervalIsSetButTimeUnitIsNotSupported() { + void validateDoesNotThrowAnyExceptionWhenScheduledUpdateIntervalIsGreaterThanMinAllowedIntervalInTenantProfile() { int scheduledUpdateInterval = 60; int minAllowedInterval = scheduledUpdateInterval - 1; diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinatesTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinatesTest.java index c5d627c3e6..a8ee18c7d7 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinatesTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/EntityCoordinatesTest.java @@ -16,47 +16,16 @@ package org.thingsboard.server.common.data.cf.configuration.geofencing; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.NullAndEmptySource; -import org.junit.jupiter.params.provider.ValueSource; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; public class EntityCoordinatesTest { - @ParameterizedTest - @ValueSource(strings = " ") - @NullAndEmptySource - void validateShouldThrowWhenLatitudeCoordinateIsNullEmptyOrBlank(String latitudeKey) { - var entityCoordinates = new EntityCoordinates(latitudeKey, "longitude"); - assertThatThrownBy(entityCoordinates::validate) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Entity coordinates latitude key name must be specified!"); - } - - @ParameterizedTest - @ValueSource(strings = " ") - @NullAndEmptySource - void validateShouldThrowWhenLongitudeCoordinateIsNullEmptyOrBlank(String longitudeKey) { - var entityCoordinates = new EntityCoordinates("latitude", longitudeKey); - assertThatThrownBy(entityCoordinates::validate) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Entity coordinates longitude key name must be specified!"); - } - - @Test - void validateShouldPassOnMinimalValidConfig() { - var entityCoordinates = new EntityCoordinates("latitude", "longitude"); - assertThatCode(entityCoordinates::validate).doesNotThrowAnyException(); - } - @Test void validateToArgumentsMethodCallWithoutRefEntityId() { var entityCoordinates = new EntityCoordinates("xPos", "yPos"); diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java index 1a3e6b4eb1..2e9d5e4a88 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java @@ -28,7 +28,6 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; @@ -44,28 +43,7 @@ public class GeofencingCalculatedFieldConfigurationTest { } @Test - void validateShouldThrowWhenEntityCoordinatesNull() { - var cfg = new GeofencingCalculatedFieldConfiguration(); - cfg.setEntityCoordinates(null); - - assertThatThrownBy(cfg::validate) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Geofencing calculated field entity coordinates must be specified!"); - } - - @Test - void validateShouldThrowWhenZoneGroupsNull() { - var cfg = new GeofencingCalculatedFieldConfiguration(); - cfg.setEntityCoordinates(new EntityCoordinates(ENTITY_ID_LATITUDE_ARGUMENT_KEY, ENTITY_ID_LONGITUDE_ARGUMENT_KEY)); - cfg.setZoneGroups(null); - - assertThatThrownBy(cfg::validate) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Geofencing calculated field must contain at least one geofencing zone group defined!"); - } - - @Test - void validateShouldCallValidateOnEntityCoordinatesAndZoneGroups() { + void validateShouldCallValidateOnZoneGroups() { var cfg = new GeofencingCalculatedFieldConfiguration(); EntityCoordinates entityCoordinatesMock = mock(EntityCoordinates.class); cfg.setEntityCoordinates(entityCoordinatesMock); @@ -73,13 +51,11 @@ public class GeofencingCalculatedFieldConfigurationTest { cfg.setZoneGroups(Map.of("someGroupName", zoneGroupConfiguration)); cfg.validate(); - - verify(entityCoordinatesMock).validate(); verify(zoneGroupConfiguration).validate("someGroupName"); } @Test - void validateShouldCallValidateOnEntityCoordinatesAndZoneGroupsWithoutAnyExceptions() { + void validateShouldCallValidateOnZoneGroupsWithoutAnyExceptions() { var cfg = new GeofencingCalculatedFieldConfiguration(); EntityCoordinates entityCoordinatesMock = mock(EntityCoordinates.class); cfg.setEntityCoordinates(entityCoordinatesMock); @@ -93,7 +69,6 @@ public class GeofencingCalculatedFieldConfigurationTest { assertThatCode(cfg::validate).doesNotThrowAnyException(); - verify(entityCoordinatesMock).validate(); verify(zoneGroupConfigurationA).validate(zoneGroupAName); verify(zoneGroupConfigurationB).validate(zoneGroupBName); } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java index 0a5c3be166..e354a05af9 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/ZoneGroupConfigurationTest.java @@ -45,24 +45,6 @@ public class ZoneGroupConfigurationTest { .hasMessage("Name '" + name + "' is reserved and cannot be used for zone group!"); } - @ParameterizedTest - @ValueSource(strings = " ") - @NullAndEmptySource - void validateShouldThrowWhenPerimeterKeyNameIsNullEmptyOrBlank(String perimeterKeyName) { - var zoneGroupConfiguration = new ZoneGroupConfiguration(perimeterKeyName, REPORT_TRANSITION_EVENTS_AND_PRESENCE_STATUS, false); - assertThatThrownBy(() -> zoneGroupConfiguration.validate("allowedZonesGroup")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Perimeter key name must be specified for 'allowedZonesGroup' zone group!"); - } - - @Test - void validateShouldThrowWhenReportStrategyIsNull() { - var zoneGroupConfiguration = new ZoneGroupConfiguration("perimeter", null, false); - assertThatThrownBy(() -> zoneGroupConfiguration.validate("allowedZonesGroup")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Report strategy must be specified for 'allowedZonesGroup' zone group!"); - } - @ParameterizedTest @ValueSource(strings = " ") @NullAndEmptySource From 6146cdc62261d7901dbddf3b628d6063e1838243 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Tue, 7 Oct 2025 17:43:16 +0300 Subject: [PATCH 027/122] MSA tests --- .../server/msa/TestRestClient.java | 45 +++++ .../server/msa/cf/CalculatedFieldTest.java | 183 ++++++++++++++++++ 2 files changed, 228 insertions(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 0a15d6e8aa..7bca833bf4 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -266,6 +266,33 @@ public class TestRestClient { .as(ArrayNode.class); } + + public ValidatableResponse deleteEntityAttributes(EntityId entityId, AttributeScope scope, String keys) { + Map pathParams = new HashMap<>(); + pathParams.put("entityId", entityId.getId().toString()); + pathParams.put("entityType", entityId.getEntityType().name()); + pathParams.put("scope", scope.name()); + return given().spec(requestSpec) + .pathParams(pathParams) + .queryParam("keys", keys) + .delete("/api/plugins/telemetry/{entityType}/{entityId}/{scope}") + .then() + .statusCode(HTTP_OK); + } + + public ValidatableResponse deleteEntityTimeseries(EntityId entityId, String keys, boolean deleteAllDataForKeys) { + Map pathParams = new HashMap<>(); + pathParams.put("entityType", entityId.getEntityType().name()); + pathParams.put("entityId", entityId.getId().toString()); + return given().spec(requestSpec) + .pathParams(pathParams) + .queryParam("keys", keys) + .queryParam("deleteAllDataForKeys", Boolean.toString(deleteAllDataForKeys)) + .delete("/api/plugins/telemetry/{entityType}/{entityId}/timeseries/delete") + .then() + .statusCode(HTTP_OK); + } + public JsonNode getLatestTelemetry(EntityId entityId) { return given().spec(requestSpec) .get("/api/plugins/telemetry/" + entityId.getEntityType().name() + "/" + entityId.getId() + "/values/timeseries") @@ -378,6 +405,24 @@ public class TestRestClient { .as(EntityRelation.class); } + + public EntityRelation deleteEntityRelation(EntityId fromId, String relationType, EntityId toId) { + Map queryParams = new HashMap<>(); + queryParams.put("fromId", fromId.getId().toString()); + queryParams.put("fromType", fromId.getEntityType().name()); + queryParams.put("relationType", relationType); + queryParams.put("toId", toId.getId().toString()); + queryParams.put("toType", toId.getEntityType().name()); + return given().spec(requestSpec) + .queryParams(queryParams) + //.delete("/api/v2/relation?fromId={fromId}&fromType={fromType}&relationType={relationType}&toId={toId}&toType={toType}") + .delete("/api/v2/relation") + .then() + .statusCode(HTTP_OK) + .extract() + .as(EntityRelation.class); + } + public JsonNode postServerSideRpc(DeviceId deviceId, JsonNode serverRpcPayload) { return given().spec(requestSpec) .body(serverRpcPayload) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java index 5e8d367538..4993f1fb8b 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java @@ -23,6 +23,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.asset.Asset; @@ -32,6 +33,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; @@ -419,6 +421,179 @@ public class CalculatedFieldTest extends AbstractContainerTest { testRestClient.deleteCalculatedFieldIfExists(saved.getId()); } + @Test + public void testPropagationCalculatedField_withExpression() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + + // --- Arrange entities --- + String deviceToken = "propagationDeviceTokenA"; + Device device = testRestClient.postDevice(deviceToken, createDevice("Propagation Device With Expression", deviceProfileId)); + Asset asset1 = testRestClient.postAsset(createAsset("Propagated Asset 1", null)); + Asset asset2 = testRestClient.postAsset(createAsset("Propagated Asset 2", null)); + + // Create relations FROM assets TO device + EntityRelation rel1 = new EntityRelation(asset1.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + EntityRelation rel2 = new EntityRelation(asset2.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + testRestClient.postEntityRelation(rel1); + testRestClient.postEntityRelation(rel2); + + // Telemetry on device + testRestClient.postTelemetry(deviceToken, JacksonUtil.toJsonNode("{\"temperature\":12.5}")); + + // --- Build CF: PROPAGATION with expression --- + CalculatedField cf = new CalculatedField(); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.PROPAGATION); + cf.setName("Propagation CF (expr)"); + cf.setConfigurationVersion(1); + + PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setDirection(EntitySearchDirection.TO); + cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setApplyExpressionToResolvedArguments(true); + + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("t", arg)); + + cfg.setExpression("{\"testResult\": t * 2}"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + cfg.setOutput(output); + + cf.setConfiguration(cfg); + + CalculatedField saved = testRestClient.postCalculatedField(cf); + + // --- Assert propagated calculation (expression applied) --- + await().alias("propagation expr mode evaluation") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode attrs1 = testRestClient.getAttributes(asset1.getId(), SERVER_SCOPE, "testResult"); + assertThat(attrs1).isNotNull().hasSize(1); + Map m1 = intKv(attrs1); + assertThat(m1).containsEntry("testResult", 25); + + ArrayNode attrs2 = testRestClient.getAttributes(asset2.getId(), SERVER_SCOPE, "testResult"); + assertThat(attrs2).isNotNull().hasSize(1); + Map m2 = intKv(attrs2); + assertThat(m2).containsEntry("testResult", 25); + }); + + testRestClient.deleteEntityRelation(asset1.getId(), EntityRelation.CONTAINS_TYPE, device.getId()); + testRestClient.deleteEntityAttributes(asset1.getId(), SERVER_SCOPE, "testResult"); + + testRestClient.postTelemetry(deviceToken, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + // --- Assert propagated calculation (expression applied with new temperature argument and one relation removed) --- + await().alias("propagation expr mode evaluation after temperature update") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode attrs1 = testRestClient.getAttributes(asset1.getId(), SERVER_SCOPE, "testResult"); + assertThat(attrs1).isNullOrEmpty(); + + ArrayNode attrs2 = testRestClient.getAttributes(asset2.getId(), SERVER_SCOPE, "testResult"); + assertThat(attrs2).isNotNull().hasSize(1); + Map m2 = intKv(attrs2); + assertThat(m2).containsEntry("testResult", 50); + }); + + testRestClient.deleteCalculatedFieldIfExists(saved.getId()); + } + + @Test + public void testPropagationCalculatedField_withoutExpression() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + + // --- Arrange entities --- + String deviceToken = "propagationDeviceTokenB"; + Device device = testRestClient.postDevice(deviceToken, createDevice("Propagation Device Without Expression", deviceProfileId)); + Asset asset1 = testRestClient.postAsset(createAsset("Propagated Asset 3", null)); + Asset asset2 = testRestClient.postAsset(createAsset("Propagated Asset 4", null)); + + // Create relations FROM assets TO device + EntityRelation rel1 = new EntityRelation(asset1.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + EntityRelation rel2 = new EntityRelation(asset2.getId(), device.getId(), EntityRelation.CONTAINS_TYPE); + testRestClient.postEntityRelation(rel1); + testRestClient.postEntityRelation(rel2); + + // Telemetry on device + long ts = System.currentTimeMillis() - 300000L; + testRestClient.postTelemetry(deviceToken, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5}}", ts))); + + // --- Build CF: PROPAGATION without expression --- + CalculatedField cf = new CalculatedField(); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.PROPAGATION); + cf.setName("Propagation CF (args-only)"); + cf.setConfigurationVersion(1); + + PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setDirection(EntitySearchDirection.TO); + cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode + + Argument arg = new Argument(); + arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("t", arg)); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + cfg.setOutput(output); + + cf.setConfiguration(cfg); + + CalculatedField saved = testRestClient.postCalculatedField(cf); + + // --- Assert propagated calculation (arguments-only mode) --- + await().alias("propagation args-only evaluation") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + JsonNode temperature1 = testRestClient.getLatestTelemetry(asset1.getId()); + assertThat(temperature1).isNotNull(); + assertThat(temperature1.get("temperature")).isNotNull(); + assertThat(temperature1.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(temperature1.get("temperature").get(0).get("value").asText()).isEqualTo("12.5"); + + JsonNode temperature2 = testRestClient.getLatestTelemetry(asset2.getId()); + assertThat(temperature2).isNotNull(); + assertThat(temperature2.get("temperature")).isNotNull(); + assertThat(temperature2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(temperature2.get("temperature").get(0).get("value").asText()).isEqualTo("12.5"); + }); + + testRestClient.deleteEntityRelation(asset1.getId(), EntityRelation.CONTAINS_TYPE, device.getId()); + testRestClient.deleteEntityTimeseries(asset1.getId(), "temperature", true); + + // Update telemetry on device + long newTs = System.currentTimeMillis() - 300000L; + testRestClient.postTelemetry(deviceToken, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs))); + + // --- Assert propagated calculation (arguments-only mode after update) --- + await().alias("propagation args-only evaluation after temperature update") + .atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + JsonNode temperature1 = testRestClient.getLatestTelemetry(asset1.getId()); + assertThat(temperature1).isNullOrEmpty(); + + JsonNode temperature2 = testRestClient.getLatestTelemetry(asset2.getId()); + assertThat(temperature2).isNotNull(); + assertThat(temperature2.get("temperature")).isNotNull(); + assertThat(temperature2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); + assertThat(temperature2.get("temperature").get(0).get("value").asInt()).isEqualTo(25); + }); + + testRestClient.deleteCalculatedFieldIfExists(saved.getId()); + } + private CalculatedField createSimpleCalculatedField() { return createSimpleCalculatedField(device.getId()); } @@ -514,4 +689,12 @@ public class CalculatedFieldTest extends AbstractContainerTest { return m; } + private static Map intKv(ArrayNode attrs) { + Map m = new HashMap<>(); + for (JsonNode n : attrs) { + m.put(n.get("key").asText(), n.get("value").asInt()); + } + return m; + } + } From f7714c2f681d0fda69a6bc05481b156e18616fd7 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Wed, 8 Oct 2025 11:17:52 +0300 Subject: [PATCH 028/122] Added output key support for Arguments only mode --- .../PropagationCalculatedFieldState.java | 6 ++--- .../cf/CalculatedFieldIntegrationTest.java | 26 +++++++++---------- .../PropagationCalculatedFieldStateTest.java | 2 +- .../server/msa/cf/CalculatedFieldTest.java | 22 ++++++++-------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java index 23c4af5bce..fed6786881 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -83,15 +83,15 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState .type(output.getType()) .scope(output.getScope()); ObjectNode valuesNode = JacksonUtil.newObjectNode(); - arguments.forEach((argumentName, argumentEntry) -> { + arguments.forEach((outputKey, argumentEntry) -> { if (argumentEntry instanceof PropagationArgumentEntry) { return; } if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) { - JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue()); + JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey); return; } - throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + argumentName + ". " + + throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + outputKey + ". " + "Only Latest telemetry or Attribute arguments supported for 'Arguments Only' propagation mode!"); }); ObjectNode result = toSimpleResult(output.getType() == OutputType.TIME_SERIES, valuesNode); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 92e4438ff3..86677ad66b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -1058,7 +1058,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument arg = new Argument(); arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); - cfg.setArguments(Map.of("t", arg)); + cfg.setArguments(Map.of("temperatureComputed", arg)); Output output = new Output(); output.setType(OutputType.TIME_SERIES); @@ -1073,14 +1073,14 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes .atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperature"); - ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperature"); + ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed"); + ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed"); assertThat(telemetry1).isNotNull(); assertThat(telemetry2).isNotNull(); - assertThat(telemetry1.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); - assertThat(telemetry1.get("temperature").get(0).get("value").asDouble()).isEqualTo(12.5); - assertThat(telemetry2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); - assertThat(telemetry2.get("temperature").get(0).get("value").asDouble()).isEqualTo(12.5); + assertThat(telemetry1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(telemetry1.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5); + assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5); }); String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", @@ -1088,7 +1088,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE ); doDelete(deleteUrl).andExpect(status().isOk()); - doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperature&deleteAllDataForKeys=true").andExpect(status().isOk()); + doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed&deleteAllDataForKeys=true").andExpect(status().isOk()); // Update telemetry on device long newTs = System.currentTimeMillis() - 300000L; @@ -1099,13 +1099,13 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes .atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperature"); - ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperature"); + ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed"); + ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed"); assertThat(telemetry1).isNotNull(); assertThat(telemetry2).isNotNull(); - assertThat(telemetry1.get("temperature").get(0).get("value")).isEqualTo(NullNode.instance); - assertThat(telemetry2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); - assertThat(telemetry2.get("temperature").get(0).get("value").asDouble()).isEqualTo(25); + assertThat(telemetry1.get("temperatureComputed").get(0).get("value")).isEqualTo(NullNode.instance); + assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); + assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25); }); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java index 2e9b3d46c9..04a7ab5203 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java @@ -179,7 +179,7 @@ public class PropagationCalculatedFieldStateTest { assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); ObjectNode expectedNode = JacksonUtil.newObjectNode(); - JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue()); + JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME); assertThat(result.getResult()).isEqualTo(expectedNode); } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java index 4993f1fb8b..8046e0b1a6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/cf/CalculatedFieldTest.java @@ -541,7 +541,7 @@ public class CalculatedFieldTest extends AbstractContainerTest { Argument arg = new Argument(); arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); - cfg.setArguments(Map.of("t", arg)); + cfg.setArguments(Map.of("temperatureComputed", arg)); Output output = new Output(); output.setType(OutputType.TIME_SERIES); @@ -558,19 +558,19 @@ public class CalculatedFieldTest extends AbstractContainerTest { .untilAsserted(() -> { JsonNode temperature1 = testRestClient.getLatestTelemetry(asset1.getId()); assertThat(temperature1).isNotNull(); - assertThat(temperature1.get("temperature")).isNotNull(); - assertThat(temperature1.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); - assertThat(temperature1.get("temperature").get(0).get("value").asText()).isEqualTo("12.5"); + assertThat(temperature1.get("temperatureComputed")).isNotNull(); + assertThat(temperature1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(temperature1.get("temperatureComputed").get(0).get("value").asText()).isEqualTo("12.5"); JsonNode temperature2 = testRestClient.getLatestTelemetry(asset2.getId()); assertThat(temperature2).isNotNull(); - assertThat(temperature2.get("temperature")).isNotNull(); - assertThat(temperature2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); - assertThat(temperature2.get("temperature").get(0).get("value").asText()).isEqualTo("12.5"); + assertThat(temperature2.get("temperatureComputed")).isNotNull(); + assertThat(temperature2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); + assertThat(temperature2.get("temperatureComputed").get(0).get("value").asText()).isEqualTo("12.5"); }); testRestClient.deleteEntityRelation(asset1.getId(), EntityRelation.CONTAINS_TYPE, device.getId()); - testRestClient.deleteEntityTimeseries(asset1.getId(), "temperature", true); + testRestClient.deleteEntityTimeseries(asset1.getId(), "temperatureComputed", true); // Update telemetry on device long newTs = System.currentTimeMillis() - 300000L; @@ -586,9 +586,9 @@ public class CalculatedFieldTest extends AbstractContainerTest { JsonNode temperature2 = testRestClient.getLatestTelemetry(asset2.getId()); assertThat(temperature2).isNotNull(); - assertThat(temperature2.get("temperature")).isNotNull(); - assertThat(temperature2.get("temperature").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); - assertThat(temperature2.get("temperature").get(0).get("value").asInt()).isEqualTo(25); + assertThat(temperature2.get("temperatureComputed")).isNotNull(); + assertThat(temperature2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); + assertThat(temperature2.get("temperatureComputed").get(0).get("value").asInt()).isEqualTo(25); }); testRestClient.deleteCalculatedFieldIfExists(saved.getId()); From 8a95f2399a61922baf1f7eef169cf66a0b516add Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 8 Oct 2025 17:14:45 +0300 Subject: [PATCH 029/122] Alarm rules CF: fixes for config update handling; fix schedule parsing --- ...CalculatedFieldEntityMessageProcessor.java | 1 - ...alculatedFieldManagerMessageProcessor.java | 6 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 18 +++---- .../alarm/AlarmCalculatedFieldState.java | 16 +++--- .../cf/ctx/state/alarm/AlarmRuleState.java | 54 +++++++++++++++---- 5 files changed, 64 insertions(+), 31 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 0a175b4899..3e9502bfe9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -155,7 +155,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM initState(state, ctx); } else { state.setCtx(ctx, actorCtx); - state.init(); } if (state.isSizeOk()) { processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d29fab508b..1f6adf2ded 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -330,11 +330,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware StateAction stateAction; if (newCfCtx.getCfType() != oldCfCtx.getCfType()) { - stateAction = StateAction.RECREATE; + stateAction = StateAction.RECREATE; // completely recreate state, then calculate } else if (newCfCtx.hasStateChanges(oldCfCtx)) { - stateAction = StateAction.REINIT; + stateAction = StateAction.REINIT; // refetch arguments, call state.init, then calculate } else if (newCfCtx.hasContextOnlyChanges(oldCfCtx)) { - stateAction = StateAction.REPROCESS; + stateAction = StateAction.REPROCESS; // call state.setCtx, then calculate } else { callback.onSuccess(); return; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index db755d3e17..fe63a76a30 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -63,8 +63,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @Data @@ -479,11 +479,11 @@ public class CalculatedFieldCtx { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } - public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { // has changes that do not require state reinit and will be picked up by the state on the fly + public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !expression.equals(other.expression)) { return true; } - if (!output.equals(other.output)) { + if (!Objects.equals(output, other.output)) { return true; } if (cfType == CalculatedFieldType.ALARM && !calculatedField.getName().equals(other.getCalculatedField().getName())) { @@ -495,9 +495,8 @@ public class CalculatedFieldCtx { return false; } - public boolean hasStateChanges(CalculatedFieldCtx other) { // has changes that require state reinit (will trigger state.reset() and re-fetch arguments) - boolean hasChanges = !arguments.equals(other.arguments); - if (hasChanges) { + public boolean hasStateChanges(CalculatedFieldCtx other) { + if (!arguments.equals(other.arguments)) { return true; } if (cfType == CalculatedFieldType.ALARM) { @@ -505,14 +504,13 @@ public class CalculatedFieldCtx { var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { - hasChanges = true; + return true; } - // TODO: implement rules update logic! } if (hasGeofencingZoneGroupConfigurationChanges(other)) { - hasChanges = true; + return true; } - return hasChanges; + return false; } private boolean hasGeofencingZoneGroupConfigurationChanges(CalculatedFieldCtx other) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 838cb2779d..6f9f59584c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -77,8 +77,8 @@ import static org.thingsboard.server.service.cf.ctx.state.alarm.AlarmEvalResult. @Slf4j public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { - private String alarmType; private AlarmCalculatedFieldConfiguration configuration; + private String alarmType; @Getter private final Map createRuleStates = new TreeMap<>(Comparator.comparing(Enum::ordinal)); @@ -97,8 +97,13 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { super.setCtx(ctx, actorCtx); - this.alarmType = ctx.getCalculatedField().getName(); this.configuration = getConfiguration(ctx); + this.alarmType = ctx.getCalculatedField().getName(); + + if (currentAlarm != null && !currentAlarm.getType().equals(alarmType)) { + currentAlarm = null; + initialFetchDone = false; + } } @Override @@ -170,10 +175,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public void reset() { super.reset(); - createRuleStates.values().forEach(AlarmRuleState::clear); - if (clearRuleState != null) { - clearRuleState.clear(); - } + configuration = null; } @Override @@ -502,7 +504,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { SingleValueArgumentEntry entry = getArgument(argument); value = mapper.apply(entry.getKvEntryValue()); if (value == null) { - throw new IllegalArgumentException("No value found for argument " + argument); + throw new IllegalArgumentException("No proper value found for argument " + argument); } } return value; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 0638543685..97bf5c5453 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -15,10 +15,11 @@ */ package org.thingsboard.server.service.cf.ctx.state.alarm; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; -import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmCondition; @@ -28,6 +29,8 @@ import org.thingsboard.server.common.data.alarm.rule.condition.DurationAlarmCond import org.thingsboard.server.common.data.alarm.rule.condition.RepeatingAlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmScheduleType; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AnyTimeSchedule; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeSchedule; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.CustomTimeScheduleItem; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.SpecificTimeSchedule; @@ -109,8 +112,12 @@ public class AlarmRuleState { eventCount++; } long requiredRepeats = getIntValue(((RepeatingAlarmCondition) condition).getCount()); - long leftRepeats = requiredRepeats - eventCount; - return leftRepeats <= 0 ? AlarmEvalResult.TRUE : AlarmEvalResult.notYetTrue(leftRepeats, 0); + if (requiredRepeats > 0) { + long leftRepeats = requiredRepeats - eventCount; + return leftRepeats <= 0 ? AlarmEvalResult.TRUE : AlarmEvalResult.notYetTrue(leftRepeats, 0); + } else { + return AlarmEvalResult.NOT_YET_TRUE; + } } else { return AlarmEvalResult.FALSE; } @@ -132,11 +139,15 @@ public class AlarmRuleState { } duration = lastEventTs - firstEventTs; long requiredDuration = getRequiredDurationInMs(); - long leftDuration = requiredDuration - duration; - if (leftDuration <= 0) { - return AlarmEvalResult.TRUE; + if (requiredDuration > 0) { + long leftDuration = requiredDuration - duration; + if (leftDuration <= 0) { + return AlarmEvalResult.TRUE; + } else { + return AlarmEvalResult.notYetTrue(0, leftDuration); + } } else { - return AlarmEvalResult.notYetTrue(0, leftDuration); + return AlarmEvalResult.NOT_YET_TRUE; } } else { return AlarmEvalResult.FALSE; @@ -148,13 +159,14 @@ public class AlarmRuleState { return true; } AlarmSchedule schedule = state.resolveValue(condition.getSchedule(), entry -> Optional.ofNullable(KvUtil.getStringValue(entry)) - .map(str -> JsonConverter.parse(str, AlarmSchedule.class)) - .orElse(null)); - return switch (schedule.getType()) { + .map(this::parseSchedule).orElse(null)); + boolean active = switch (schedule.getType()) { case ANY_TIME -> true; case SPECIFIC_TIME -> isActiveSpecific((SpecificTimeSchedule) schedule, eventTs); case CUSTOM -> isActiveCustom((CustomTimeSchedule) schedule, eventTs); }; + log.trace("Alarm rule active = {} for schedule {}", active, schedule); + return active; } private boolean isActiveSpecific(SpecificTimeSchedule schedule, long eventTs) { @@ -221,6 +233,28 @@ public class AlarmRuleState { return eventCount == 0L && firstEventTs == 0L && lastEventTs == 0L && durationCheckFuture == null; } + private AlarmSchedule parseSchedule(String str) { + ObjectNode json = (ObjectNode) JacksonUtil.toJsonNode(str); + if (json.isEmpty()) { + return new AnyTimeSchedule(); // only if valid json, fail otherwise + } + + if (!json.hasNonNull("type")) { + // deducting the schedule type + AlarmScheduleType type; + if (json.hasNonNull("daysOfWeek")) { + type = AlarmScheduleType.SPECIFIC_TIME; + } else if (json.hasNonNull("items")) { + type = AlarmScheduleType.CUSTOM; + } else { + throw new IllegalArgumentException("Failed to parse alarm schedule from '" + str + "'"); + } + json.put("type", type.name()); + } + + return JacksonUtil.treeToValue(json, AlarmSchedule.class); + } + private Integer getIntValue(AlarmConditionValue value) { return state.resolveValue(value, entry -> Optional.ofNullable(KvUtil.getLongValue(entry)).map(Long::intValue).orElse(null)); } From 8c933b85f15ed549b3f1f53eb24ce42b59078257 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 8 Oct 2025 17:14:58 +0300 Subject: [PATCH 030/122] Alarm rules CF: more tests --- .../thingsboard/server/cf/AlarmRulesTest.java | 343 ++++++++++++++++-- 1 file changed, 311 insertions(+), 32 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 830fe5559b..31772b3089 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.cf; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -43,6 +44,7 @@ import org.thingsboard.server.common.data.alarm.rule.condition.expression.predic import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate.NumericOperation; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate.StringOperation; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; @@ -57,7 +59,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -65,6 +66,7 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -95,7 +97,7 @@ public class AlarmRulesTest extends AbstractControllerTest { } @Test - public void testCreateAndSeverityUpdateAndClear() throws Exception { + public void testCreateAlarm_severityUpdate_clear() throws Exception { Argument temperatureArgument = new Argument(); temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); temperatureArgument.setDefaultValue("0"); @@ -111,8 +113,6 @@ public class AlarmRulesTest extends AbstractControllerTest { Condition clearRule = new Condition("return temperature <= 25;", null, null); CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", arguments, createRules, clearRule); - assertThat(getCalculatedFields(deviceId, CalculatedFieldType.ALARM, new PageLink(1)).getData()) - .singleElement().isEqualTo(calculatedField); postTelemetry(deviceId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { @@ -178,11 +178,8 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } - /* - * todo: test state restore (event count) - * */ @Test - public void testCreateAlarmForRepeatingCondition() throws Exception { + public void testCreateAlarm_repeatingCondition() throws Exception { Argument temperatureArgument = new Argument(); temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); temperatureArgument.setDefaultValue("0"); @@ -225,7 +222,42 @@ public class AlarmRulesTest extends AbstractControllerTest { } @Test - public void testCreateAlarmForDurationCondition() throws Exception { + public void testCreateAlarm_dynamicRepeatingCondition() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + + Argument eventsCountArgument = new Argument(); + eventsCountArgument.setRefEntityKey(new ReferencedEntityKey("eventsCount", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + eventsCountArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument, + "eventsCount", eventsCountArgument + ); + + int eventsCount = 5; + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", null, + new AlarmConditionValue<>(null, "eventsCount"), null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"eventsCount\":" + eventsCount + "}"); + for (int i = 0; i < eventsCount; i++) { + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(10); + } + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(eventsCount); + }); + } + + @Test + public void testCreateAlarm_durationCondition() throws Exception { Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("powerConsumption", ArgumentType.TS_LATEST, null)); argument.setDefaultValue("0"); @@ -234,10 +266,11 @@ public class AlarmRulesTest extends AbstractControllerTest { ); long createDurationMs = 5000L; + long clearDurationMs = 3000L; Map createRules = Map.of( AlarmSeverity.CRITICAL, new Condition("return powerConsumption >= 3000;", null, createDurationMs) ); - Condition clearRule = new Condition("return powerConsumption < 3000;", null, createDurationMs); + Condition clearRule = new Condition("return powerConsumption < 3000;", null, clearDurationMs); CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 5 seconds", arguments, createRules, clearRule); @@ -251,6 +284,49 @@ public class AlarmRulesTest extends AbstractControllerTest { assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); assertThat(alarmResult.getConditionDuration()).isBetween(createDurationMs, createDurationMs + 2000); }); + + postTelemetry(deviceId, "{\"powerConsumption\":2000}"); + Thread.sleep(clearDurationMs - 2000); + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCleared()).isTrue(); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.CLEARED_UNACK); + assertThat(alarmResult.getConditionDuration()).isBetween(clearDurationMs, clearDurationMs + 2000); + }); + } + + @Test + public void testCreateAlarm_dynamicDurationCondition() throws Exception { + Argument powerConsumptionArgument = new Argument(); + powerConsumptionArgument.setRefEntityKey(new ReferencedEntityKey("powerConsumption", ArgumentType.TS_LATEST, null)); + powerConsumptionArgument.setDefaultValue("0"); + + Argument durationArgument = new Argument(); + durationArgument.setRefEntityKey(new ReferencedEntityKey("duration", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + durationArgument.setDefaultValue("-1"); + Map arguments = Map.of( + "powerConsumption", powerConsumptionArgument, + "duration", durationArgument + ); + + long createDurationMs = 2000L; + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return powerConsumption >= 3000;", null, null, + new AlarmConditionValue(null, "duration"), null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High power consumption during 2 seconds", + arguments, createRules, null); + postTelemetry(deviceId, "{\"powerConsumption\":3500}"); + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"duration\":" + createDurationMs + "}"); + + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionDuration()).isBetween(createDurationMs, createDurationMs + 2000); + }); } @Test @@ -334,18 +410,212 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } - private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { - await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { - TbAlarmResult alarmResult = getLatestAlarmResult(calculatedField.getId()); - assertThat(alarmResult).isNotNull(); - assertion.accept(alarmResult); - - Alarm alarm = alarmResult.getAlarm(); - assertThat(alarm.getOriginator()).isEqualTo(originatorId); - assertThat(alarm.getType()).isEqualTo(calculatedField.getName()); + @Test + public void testCreateAlarm_dynamicSchedule() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Argument scheduleArgument = new Argument(); + scheduleArgument.setRefEntityKey(new ReferencedEntityKey("schedule", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + scheduleArgument.setDefaultValue("None"); + Map arguments = Map.of( + "temperature", temperatureArgument, + "schedule", scheduleArgument // fixme: + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", null, null, null, + new AlarmConditionValue<>(null, "schedule")) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + String schedule = """ + {"timezone":"Europe/Kiev","items":[{"enabled":false,"dayOfWeek":1,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":2,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":3,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":4,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":5,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":6,"startsOn":0,"endsOn":0},{"enabled":false,"dayOfWeek":7,"startsOn":0,"endsOn":0}]} + """; + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"schedule\":" + schedule + "}"); + postTelemetry(deviceId, "{\"temperature\":50}"); + + Thread.sleep(1000); + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + + schedule = schedule.replace("\"enabled\":false", "\"enabled\":true"); + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"schedule\":" + schedule + "}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + + @Test + public void testChangeAlarmType() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + + postTelemetry(deviceId, "{\"temperature\":50}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + + calculatedField.setName("New alarm type"); + calculatedField = saveCalculatedField(calculatedField); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + + @Test + public void testChangeRuleExpression() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 100;", null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(1000); + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + + AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + ((TbelAlarmConditionExpression) configuration.getCreateRules().get(AlarmSeverity.CRITICAL).getCondition().getExpression()) + .setExpression("return temperature >= 50;"); + calculatedField = saveCalculatedField(calculatedField); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + + @Test + public void testChangeRequiredEventsCountForRepeatingCondition() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + int eventsCountMajor = 5; + int eventsCountCritical = 10; + Map createRules = Map.of( + AlarmSeverity.MAJOR, new Condition("return temperature >= 50;", eventsCountMajor, null), + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", eventsCountCritical, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + for (int i = 0; i < eventsCountMajor; i++) { + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(10); + } + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(5); + }); + + postTelemetry(deviceId, "{\"temperature\":50}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(6); + }); + + // decreasing required events count for critical rule + AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + ((RepeatingAlarmCondition) configuration.getCreateRules().get(AlarmSeverity.CRITICAL).getCondition()) + .setCount(new AlarmConditionValue<>(6, null)); + calculatedField = saveCalculatedField(calculatedField); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isSeverityUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + assertThat(alarmResult.getConditionRepeats()).isEqualTo(6); }); } + @Test + public void testChangeConditionArgumentSource() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + + Argument temperatureThresholdArgument = new Argument(); + temperatureThresholdArgument.setRefEntityKey(new ReferencedEntityKey("temperatureThreshold", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + temperatureThresholdArgument.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + temperatureThresholdArgument.setDefaultValue("100"); + loginSysAdmin(); + postAttributes(tenantId, AttributeScope.SERVER_SCOPE, "{\"temperatureThreshold\":100}"); + loginTenantAdmin(); + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"temperatureThreshold\":50}"); + + Map arguments = Map.of( + "temperature", temperatureArgument, + "temperatureThreshold", temperatureThresholdArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= temperatureThreshold;", null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + + postTelemetry(deviceId, "{\"temperature\":50}"); + Thread.sleep(1000); + // not created because tenant's threshold 100 is used + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + + ((AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration()).getArguments().get("temperatureThreshold") + .setRefDynamicSourceConfiguration(null); + // using threshold 50 on device level + calculatedField = saveCalculatedField(calculatedField); + + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + + // todo: test alarm details + + private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { + TbAlarmResult alarmResult = await().atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getLatestAlarmResult(calculatedField.getId()), Objects::nonNull); + assertion.accept(alarmResult); + + Alarm alarm = alarmResult.getAlarm(); + assertThat(alarm.getOriginator()).isEqualTo(originatorId); + assertThat(alarm.getType()).isEqualTo(calculatedField.getName()); + } + private TbAlarmResult getLatestAlarmResult(CalculatedFieldId calculatedFieldId) { List debugEvents = getDebugEvents(calculatedFieldId, 1); if (debugEvents.isEmpty()) { @@ -410,23 +680,22 @@ public class AlarmRulesTest extends AbstractControllerTest { if (condition.getEventsCount() != null) { RepeatingAlarmCondition alarmCondition = new RepeatingAlarmCondition(); alarmCondition.setExpression(expression); - AlarmConditionValue count = new AlarmConditionValue<>(); - count.setStaticValue(condition.getEventsCount()); - alarmCondition.setCount(count); + alarmCondition.setCount(condition.getEventsCount()); rule.setCondition(alarmCondition); - } else if (condition.getDurationMs() != null) { + } else if (condition.getDuration() != null) { DurationAlarmCondition alarmCondition = new DurationAlarmCondition(); alarmCondition.setExpression(expression); alarmCondition.setUnit(TimeUnit.MILLISECONDS); - AlarmConditionValue duration = new AlarmConditionValue<>(); - duration.setStaticValue(condition.getDurationMs()); - alarmCondition.setValue(duration); + alarmCondition.setValue(condition.getDuration()); rule.setCondition(alarmCondition); } else { SimpleAlarmCondition alarmCondition = new SimpleAlarmCondition(); alarmCondition.setExpression(expression); rule.setCondition(alarmCondition); } + if (condition.getSchedule() != null) { + rule.getCondition().setSchedule(condition.getSchedule()); + } return rule; } @@ -448,25 +717,35 @@ public class AlarmRulesTest extends AbstractControllerTest { } @Getter + @AllArgsConstructor private static final class Condition { private final String tbelExpression; private final SimpleAlarmConditionExpression simpleExpression; - private final Integer eventsCount; - private final Long durationMs; + private AlarmConditionValue eventsCount; + private AlarmConditionValue duration; + private AlarmConditionValue schedule; private Condition(String tbelExpression, Integer eventsCount, Long durationMs) { this.tbelExpression = tbelExpression; this.simpleExpression = null; - this.eventsCount = eventsCount; - this.durationMs = durationMs; + if (eventsCount != null) { + this.eventsCount = new AlarmConditionValue<>(eventsCount, null); + } + if (durationMs != null) { + this.duration = new AlarmConditionValue<>(durationMs, null); + } } private Condition(SimpleAlarmConditionExpression simpleExpression, Integer eventsCount, Long durationMs) { this.tbelExpression = null; this.simpleExpression = simpleExpression; - this.eventsCount = eventsCount; - this.durationMs = durationMs; + if (eventsCount != null) { + this.eventsCount = new AlarmConditionValue<>(eventsCount, null); + } + if (durationMs != null) { + this.duration = new AlarmConditionValue<>(durationMs, null); + } } } From a12c0d070413f06d2e824be15b13a4664054ca68 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 12:17:32 +0300 Subject: [PATCH 031/122] CF: fix update of entry with default value --- .../AbstractCalculatedFieldProcessingService.java | 7 ++++--- .../cf/ctx/state/SingleValueArgumentEntry.java | 13 +++++++++++-- .../ctx/state/alarm/AlarmCalculatedFieldState.java | 2 -- .../cf/ctx/state/SingleValueArgumentEntryTest.java | 6 ++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 4018916582..d17148a502 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -43,6 +43,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.HashMap; import java.util.List; @@ -226,12 +227,12 @@ public abstract class AbstractCalculatedFieldProcessingService { return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); return transformSingleValueArgument(Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } - protected ListenableFuture fetchTsLatest(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + protected ListenableFuture fetchTsLatest(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { String timeseriesKey = argument.getRefEntityKey().getKey(); log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, timeseriesKey); return transformSingleValueArgument( @@ -239,7 +240,7 @@ public abstract class AbstractCalculatedFieldProcessingService { timeseriesService.findLatest(tenantId, entityId, timeseriesKey), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, timeseriesKey, result); - return result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); + return result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); }, calculatedFieldCallbackExecutor)); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 1ceea2c621..5c1ed32e1d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -43,6 +43,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private boolean forceResetPrevious; + public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { @@ -112,8 +114,10 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() <= this.ts) { - return false; + if (singleValueEntry.getTs() < this.ts) { + if (!isDefaultValue()) { + return false; + } } Long newVersion = singleValueEntry.getVersion(); @@ -128,4 +132,9 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } return false; } + + public boolean isDefaultValue() { + return DEFAULT_VERSION.equals(this.version); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 6f9f59584c..bc9ed4e32e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -300,8 +300,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { private TbAlarmResult calculateAlarmResult(AlarmRuleState ruleState, CalculatedFieldCtx ctx) { AlarmSeverity severity = ruleState.getSeverity(); if (currentAlarm != null) { - // TODO: In some extremely rare cases, we might miss the event of alarm clear (If one use in-mem queue and restarted the server) or (if one manipulated the rule chain). - // Maybe we should fetch alarm every time? currentAlarm.setEndTs(System.currentTimeMillis()); AlarmSeverity oldSeverity = currentAlarm.getSeverity(); // Skip update if severity is decreased. diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 5ea3808468..4ada355054 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -57,6 +57,11 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } + @Test + void testUpdateEntryWithTheSameTsAndDifferentVersion() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 364L))).isTrue(); + } + @Test void testUpdateEntryWhenNewVersionIsNull() { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); @@ -115,4 +120,5 @@ public class SingleValueArgumentEntryTest { expectedList.add(Map.of("test2", 20)); assertThat(singleValueArg.getValue()).isEqualTo(expectedList); } + } From 728d1f78b08141635be1b7863a60b706cb1e606f Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 12:17:47 +0300 Subject: [PATCH 032/122] Alarm rules CF: add test for alarm details --- .../thingsboard/server/cf/AlarmRulesTest.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 31772b3089..f468bd709b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -604,7 +604,41 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } - // todo: test alarm details + @Test + public void testAlarmDetails() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Argument humidityArgument = new Argument(); + humidityArgument.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + humidityArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument, + "humidity", humidityArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50 && humidity >= 50;", null, null) + ); + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature and Humidity Alarm", + arguments, createRules, null); + AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + configuration.getCreateRules().get(AlarmSeverity.CRITICAL).setAlarmDetails(""" + temperature is ${temperature}, humidity is ${humidity}"""); + calculatedField = saveCalculatedField(calculatedField); + + postTelemetry(deviceId, "{\"temperature\":50}"); + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"humidity\":50}"); + + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getDetails().get("data").asText()) + .isEqualTo("temperature is 50, humidity is 50"); + }); + } + + // TODO: MSA tests private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { TbAlarmResult alarmResult = await().atMost(TIMEOUT, TimeUnit.SECONDS) From 9f208b4abd7628de30f633493bece143fb6b7684 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 9 Oct 2025 13:02:02 +0300 Subject: [PATCH 033/122] Added related entities limit + additional validation for CF configuration --- .../main/data/upgrade/basic/schema_update.sql | 10 +- .../cf/ctx/state/CalculatedFieldCtx.java | 8 +- ...opagationCalculatedFieldConfiguration.java | 3 + .../DefaultTenantProfileConfiguration.java | 4 +- ...ationCalculatedFieldConfigurationTest.java | 24 ++++ .../dao/relation/BaseRelationService.java | 15 ++- .../server/dao/relation/RelationDao.java | 2 +- .../dao/sql/relation/JpaRelationDao.java | 20 ++- .../dao/service/RelationServiceTest.java | 117 ++++++++++++++---- 9 files changed, 160 insertions(+), 43 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 0add4c0545..50c44c0e2d 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -34,7 +34,13 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' THEN NULL ELSE to_jsonb(10) - END + END, + 'maxRelatedEntitiesToReturnPerCfArgument', + CASE + WHEN (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument' + THEN NULL + ELSE to_jsonb(100) + END, ) ), false @@ -43,6 +49,8 @@ WHERE NOT ( (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF' AND (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' + AND + (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index bfb6483d18..5b6b708d26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -498,12 +498,8 @@ public class CalculatedFieldCtx { } public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { // has changes that do not require state reinit and will be picked up by the state on the fly - if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration expressionConfig) { - boolean shouldCompareExpression = !(expressionConfig instanceof PropagationCalculatedFieldConfiguration propagationConfig) - || propagationConfig.isApplyExpressionToResolvedArguments(); - if (shouldCompareExpression && !expression.equals(other.expression)) { - return true; - } + if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !Objects.equals(expression, other.expression)) { + return true; } if (!output.equals(other.output)) { return true; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index 3394813d3f..70ffd952cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -50,6 +50,9 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField propagationRestriction(); if (!applyExpressionToResolvedArguments) { arguments.forEach((name, argument) -> { + if (argument.getRefEntityId() != null || argument.getRefDynamicSourceConfiguration() != null) { + throw new IllegalArgumentException("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } if (argument.getRefEntityKey() == null) { throw new IllegalArgumentException("Argument: '" + name + "' doesn't have reference entity key configured!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index c6bd9a7f38..5d1c4f7018 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -172,10 +172,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxCalculatedFieldsPerEntity = 5; @Schema(example = "10") private long maxArgumentsPerCF = 10; - @Schema(example = "3600") + @Schema(example = "60") private int minAllowedScheduledUpdateIntervalInSecForCF = 60; @Schema(example = "10") private int maxRelationLevelPerCfArgument = 10; + @Schema(example = "100") + private int maxRelatedEntitiesToReturnPerCfArgument = 100; @Builder.Default @Min(value = 1, message = "must be at least 1") @Schema(example = "1000") diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java index eb0591e32b..64f7b0efd0 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java @@ -19,10 +19,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import java.util.Map; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -37,6 +39,28 @@ public class PropagationCalculatedFieldConfigurationTest { assertThat(cfg.getType()).isEqualTo(CalculatedFieldType.PROPAGATION); } + @Test + void validateShouldThrowWhenConfigurationDisallowArgumentsWithReferencedEntity() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argumentWithRefEntityIdSet = new Argument(); + argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("bda14084-f40e-4acc-9b85-9d1dd209bb64"))); + cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } + + @Test + void validateShouldThrowWhenConfigurationDisallowArgumentsWithDynamicReferenceConfiguration() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argumentWithRefEntityIdSet = new Argument(); + argumentWithRefEntityIdSet.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } + @Test void validateShouldThrowWhenUsedReservedPropagationArgumentName() { var cfg = new PropagationCalculatedFieldConfiguration(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..34928c7e04 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -50,12 +50,14 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.relation.JpaRelationQueryExecutorService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import java.util.ArrayList; import java.util.Collections; @@ -71,6 +73,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePositiveNumber; /** * Created by ashvayka on 28.04.17. @@ -85,6 +88,8 @@ public class BaseRelationService implements RelationService { private final ApplicationEventPublisher eventPublisher; private final JpaExecutorService executor; private final JpaRelationQueryExecutorService relationsExecutor; + private final ApiLimitService apiLimitService; + protected ScheduledExecutorService timeoutExecutorService; @Value("${sql.relations.query_timeout:20}") @@ -93,13 +98,14 @@ public class BaseRelationService implements RelationService { public BaseRelationService(RelationDao relationDao, @Lazy EntityService entityService, TbTransactionalCache cache, ApplicationEventPublisher eventPublisher, JpaExecutorService executor, - JpaRelationQueryExecutorService relationsExecutor) { + JpaRelationQueryExecutorService relationsExecutor, ApiLimitService apiLimitService) { this.relationDao = relationDao; this.entityService = entityService; this.cache = cache; this.eventPublisher = eventPublisher; this.executor = executor; this.relationsExecutor = relationsExecutor; + this.apiLimitService = apiLimitService; } @PostConstruct @@ -504,14 +510,17 @@ public class BaseRelationService implements RelationService { log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); validateId(tenantId, id -> "Invalid tenant id: " + id); validate(relationPathQuery); + int limit = (int) apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxRelatedEntitiesToReturnPerCfArgument); + validatePositiveNumber(limit, "Invalid entities limit: " + limit); if (relationPathQuery.levels().size() == 1) { RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); - return switch (relationPathLevel.direction()) { + var relationsFuture = switch (relationPathLevel.direction()) { case FROM -> findByFromAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); case TO -> findByToAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); }; + return Futures.transform(relationsFuture, entityRelations -> entityRelations.subList(0, limit), MoreExecutors.directExecutor()); } - return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); + return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery, limit)); } private void validate(EntityRelationPathQuery relationPathQuery) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..2ec23a0d74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -72,6 +72,6 @@ public interface RelationDao { List findRuleNodeToRuleChainRelations(RuleChainType ruleChainType, int limit); - List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery, int limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..4c1b8f299b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -299,15 +299,18 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query) { + public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query, int limit) { List levels = query.levels(); if (levels == null || levels.isEmpty()) { - return Collections.emptyList(); + return List.of(); + } + if (limit <= 0) { + return List.of(); } String sql = buildRelationPathSql(query); - Object[] params = buildRelationPathParams(query); + Object[] params = buildRelationPathParams(query, limit); - log.trace("[{}] relation path query: {}", tenantId, sql); + log.info("[{}] relation path query: {}", tenantId, sql); return jdbcTemplate.queryForList(sql, params).stream() .map(row -> { @@ -330,7 +333,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple .collect(Collectors.toList()); } - private Object[] buildRelationPathParams(EntityRelationPathQuery query) { + private Object[] buildRelationPathParams(EntityRelationPathQuery query, int limit) { final List params = new ArrayList<>(); // seed params.add(query.rootEntityId().getId()); @@ -340,6 +343,10 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple for (var lvl : query.levels()) { params.add(lvl.relationType()); } + + // limit + params.add(limit); + return params.toArray(); } @@ -387,7 +394,8 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n") .append("JOIN ").append(prevForLast).append(" p ON ").append(lastJoin).append("\n") .append("WHERE r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n") - .append(" AND r.relation_type = ?"); + .append(" AND r.relation_type = ?\n") + .append("LIMIT ?"); return sb.toString(); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java index bb7bfad677..e3827d6a6e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -52,6 +53,9 @@ public class RelationServiceTest extends AbstractServiceTest { @Autowired RelationService relationService; + @Autowired + private TbTenantProfileCache tbTenantProfileCache; + @Before public void before() { } @@ -628,48 +632,111 @@ public class RelationServiceTest extends AbstractServiceTest { } @Test - public void testFindByPathQuery() throws Exception { + public void testFindByPathQueryWithoutExceedingLimit() throws Exception { /* A └──[firstLevel, TO]→ B └──[secondLevel, TO]→ C - ├──[thirdLevel, FROM]→ D - ├──[thirdLevel, FROM]→ E - └──[thirdLevel, FROM]→ F + ├──[thirdLevel, FROM]→ D1 + ├──[thirdLevel, FROM]→ D2 + ├──[thirdLevel, FROM]→ ... + └──[thirdLevel, FROM]→ D{N - 1}, where N is the limit */ - // rootEntity AssetId assetA = new AssetId(Uuids.timeBased()); - // firstLevelEntity AssetId assetB = new AssetId(Uuids.timeBased()); - // secondLevelEntity AssetId assetC = new AssetId(Uuids.timeBased()); - // thirdLevelEntities - AssetId assetD = new AssetId(Uuids.timeBased()); - AssetId assetE = new AssetId(Uuids.timeBased()); - AssetId assetF = new AssetId(Uuids.timeBased()); - EntityRelation firstLevelRelation = new EntityRelation(assetB, assetA, "firstLevel"); - EntityRelation secondLevelRelation = new EntityRelation(assetC, assetB, "secondLevel"); - EntityRelation thirdLevelRelation1 = new EntityRelation(assetC, assetD, "thirdLevel"); - EntityRelation thirdLevelRelation2 = new EntityRelation(assetC, assetE, "thirdLevel"); - EntityRelation thirdLevelRelation3 = new EntityRelation(assetC, assetF, "thirdLevel"); + // create first and second level + saveRelation(new EntityRelation(assetB, assetA, "firstLevel")); + saveRelation(new EntityRelation(assetC, assetB, "secondLevel")); - firstLevelRelation = saveRelation(firstLevelRelation); - secondLevelRelation = saveRelation(secondLevelRelation); - thirdLevelRelation1 = saveRelation(thirdLevelRelation1); - thirdLevelRelation2 = saveRelation(thirdLevelRelation2); - thirdLevelRelation3 = saveRelation(thirdLevelRelation3); + int limit = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMaxRelatedEntitiesToReturnPerCfArgument(); - List expectedRelations = List.of(thirdLevelRelation1, thirdLevelRelation2, thirdLevelRelation3); + int totalCreated = limit - 1; - EntityRelationPathQuery relationPathQuery = new EntityRelationPathQuery(assetA, List.of( + List allThirdLevelRelations = new ArrayList<>(); + for (int i = 0; i < totalCreated; i++) { + AssetId leaf = new AssetId(Uuids.timeBased()); + allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel"))); + } + + EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of( new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"), new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"), new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel") )); - List entityRelations = relationService.findByRelationPathQueryAsync(tenantId, relationPathQuery).get(); - assertThat(expectedRelations).containsExactlyInAnyOrderElementsOf(entityRelations); + // call a method that applies the default limit internally + List result = relationService.findByRelationPathQueryAsync(tenantId, query).get(); + + // verify that limit has been applied + assertThat(result).hasSize(totalCreated); + + // verify all returned are valid third-level relations under C + assertThat(result) + .allSatisfy(rel -> { + assertThat(rel.getType()).isEqualTo("thirdLevel"); + assertThat(rel.getFrom()).isEqualTo(assetC); + }); + + // verify the returned subset is part of all created relations + assertThat(result).isEqualTo(allThirdLevelRelations); + } + + @Test + public void testFindByPathQueryWithExceedingLimit() throws Exception { + /* + A + └──[firstLevel, TO]→ B + └──[secondLevel, TO]→ C + ├──[thirdLevel, FROM]→ D1 + ├──[thirdLevel, FROM]→ D2 + ├──[thirdLevel, FROM]→ ... + └──[thirdLevel, FROM]→ D{N + 20}, where N is the limit + */ + AssetId assetA = new AssetId(Uuids.timeBased()); + AssetId assetB = new AssetId(Uuids.timeBased()); + AssetId assetC = new AssetId(Uuids.timeBased()); + + // create first and second level + saveRelation(new EntityRelation(assetB, assetA, "firstLevel")); + saveRelation(new EntityRelation(assetC, assetB, "secondLevel")); + + int limit = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMaxRelatedEntitiesToReturnPerCfArgument(); + + int totalCreated = limit + 20; + + List allThirdLevelRelations = new ArrayList<>(); + for (int i = 0; i < totalCreated; i++) { + AssetId leaf = new AssetId(Uuids.timeBased()); + allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel"))); + } + + EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of( + new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"), + new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"), + new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel") + )); + + // call a method that applies the default limit internally + List result = relationService.findByRelationPathQueryAsync(tenantId, query).get(); + + // verify that limit has been applied + assertThat(result).hasSize(limit); + + // verify all returned are valid third-level relations under C + assertThat(result) + .allSatisfy(rel -> { + assertThat(rel.getType()).isEqualTo("thirdLevel"); + assertThat(rel.getFrom()).isEqualTo(assetC); + }); + + // verify the returned subset is part of all created relations + assertThat(result).isSubsetOf(allThirdLevelRelations); } @Test From f381125d6957f79ba8dae15c9112cc630eb05a1a Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 9 Oct 2025 13:15:14 +0300 Subject: [PATCH 034/122] fix typo --- .../org/thingsboard/server/dao/sql/relation/JpaRelationDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 4c1b8f299b..3ab382d2a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -310,7 +310,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple String sql = buildRelationPathSql(query); Object[] params = buildRelationPathParams(query, limit); - log.info("[{}] relation path query: {}", tenantId, sql); + log.trace("[{}] relation path query: {}", tenantId, sql); return jdbcTemplate.queryForList(sql, params).stream() .map(row -> { From d16ee53bcd76cceb0906778473c6ba6fa51e1e28 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 13:16:19 +0300 Subject: [PATCH 035/122] CF: expressions usage refactoring --- .../cf/ctx/state/CalculatedFieldCtx.java | 27 +++++++++------- .../ctx/state/ScriptCalculatedFieldState.java | 15 +++++++-- .../ctx/state/SimpleCalculatedFieldState.java | 32 ++++++++++++------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 14634bbaab..9e9842c28d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -176,13 +176,8 @@ public class CalculatedFieldCtx { public void init() { switch (cfType) { case SCRIPT -> { - try { - initTbelExpression(expression); - initialized = true; - } catch (Exception e) { - initialized = false; - throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); - } + initTbelExpression(expression); + initialized = true; } case GEOFENCING -> initialized = true; case SIMPLE -> { @@ -205,8 +200,7 @@ public class CalculatedFieldCtx { } } - public double evaluateSimpleExpression(String expressionStr, CalculatedFieldState state) { - Expression expression = simpleExpressions.get(expressionStr).get(); + public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) { for (Map.Entry entry : state.getArguments().entrySet()) { try { BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); @@ -225,6 +219,10 @@ public class CalculatedFieldCtx { } public ListenableFuture evaluateTbelExpression(String expression, CalculatedFieldState state) { + return evaluateTbelExpression(tbelExpressions.get(expression), state); + } + + public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, CalculatedFieldState state) { Map arguments = new LinkedHashMap<>(); List args = new ArrayList<>(argNames.size() + 1); args.add(new Object()); // first element is a ctx, but we will set it later; @@ -239,7 +237,7 @@ public class CalculatedFieldCtx { } args.set(0, new TbelCfCtx(arguments, state.getLatestTimestamp())); - return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + return expression.executeScriptAsync(args.toArray()); } public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { @@ -258,8 +256,13 @@ public class CalculatedFieldCtx { } else if (tbelExpressions.containsKey(expression)) { return; } - CalculatedFieldScriptEngine engine = initEngine(tenantId, expression, tbelInvokeService); - tbelExpressions.put(expression, engine); + try { + CalculatedFieldScriptEngine engine = initEngine(tenantId, expression, tbelInvokeService); + tbelExpressions.put(expression, engine); + } catch (Exception e) { + initialized = false; + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } } private void initSimpleExpression(String expression) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 3b6f9b1f87..9c5a25fda9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.EntityId; @@ -33,18 +34,21 @@ import java.util.Map; @EqualsAndHashCode(callSuper = true) public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + private CalculatedFieldScriptEngine tbelExpression; + public ScriptCalculatedFieldState(EntityId entityId) { super(entityId); } @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.SCRIPT; + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression()); } @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { - ListenableFuture resultFuture = ctx.evaluateTbelExpression(ctx.getExpression(), this); + ListenableFuture resultFuture = ctx.evaluateTbelExpression(tbelExpression, this); Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> TelemetryCalculatedFieldResult.builder() @@ -56,4 +60,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { ); } + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 3a98fee361..65cb595632 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,8 +20,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.EqualsAndHashCode; +import net.objecthunter.exp4j.Expression; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbUtils; +import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.EntityId; @@ -33,26 +35,21 @@ import java.util.Map; @EqualsAndHashCode(callSuper = true) public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + private ThreadLocal expression; + public SimpleCalculatedFieldState(EntityId entityId) { super(entityId); } @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.SIMPLE; - } - - @Override - protected void validateNewEntry(String key, ArgumentEntry newEntry) { - if (newEntry instanceof TsRollingArgumentEntry) { - throw new IllegalArgumentException("Unsupported argument type detected for argument: " + key + ". " + - "Rolling argument entry is not supported for simple calculated fields."); - } + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + this.expression = ctx.getSimpleExpressions().get(ctx.getExpression()); } @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { - double expressionResult = ctx.evaluateSimpleExpression(ctx.getExpression(), this); + double expressionResult = ctx.evaluateSimpleExpression(expression.get(), this); Output output = ctx.getOutput(); Object result = formatResult(expressionResult, output.getDecimalsByDefault()); @@ -96,4 +93,17 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } } + @Override + protected void validateNewEntry(String key, ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Unsupported argument type detected for argument: " + key + ". " + + "Rolling argument entry is not supported for simple calculated fields."); + } + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + } From f5804d928b093d3bb1ae60a2bb83fe8ead492ccb Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 9 Oct 2025 14:51:53 +0300 Subject: [PATCH 036/122] Added validation for Expression result propagation mode --- ...opagationCalculatedFieldConfiguration.java | 20 ++++++++++++++++--- ...ationCalculatedFieldConfigurationTest.java | 17 ++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index 70ffd952cf..5e8c822d78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -50,7 +50,7 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField propagationRestriction(); if (!applyExpressionToResolvedArguments) { arguments.forEach((name, argument) -> { - if (argument.getRefEntityId() != null || argument.getRefDynamicSourceConfiguration() != null) { + if (!currentEntitySource(argument)) { throw new IllegalArgumentException("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); } if (argument.getRefEntityKey() == null) { @@ -61,8 +61,17 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField "Only 'Attribute' or 'Latest telemetry' arguments are allowed for 'Arguments only' propagation mode!"); } }); - } else if (StringUtils.isBlank(expression)) { - throw new IllegalArgumentException("Expression must be specified for 'Expression result' propagation mode!"); + } else { + boolean noneMatchCurrentEntitySource = arguments.entrySet() + .stream() + .noneMatch(entry -> currentEntitySource(entry.getValue())); + if (noneMatchCurrentEntitySource) { + throw new IllegalArgumentException("At least one argument must be configured with the 'Current entity' " + + "source entity type for 'Expression result' propagation mode!"); + } + if (StringUtils.isBlank(expression)) { + throw new IllegalArgumentException("Expression must be specified for 'Expression result' propagation mode!"); + } } } @@ -79,4 +88,9 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField throw new IllegalArgumentException("Argument name '" + PROPAGATION_CONFIG_ARGUMENT + "' is reserved and cannot be used."); } } + + private boolean currentEntitySource(Argument argument) { + return argument.getRefEntityId() == null && argument.getRefDynamicSourceConfiguration() == null; + } + } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java index 64f7b0efd0..36f63feed7 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java @@ -52,13 +52,26 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenConfigurationDisallowArgumentsWithDynamicReferenceConfiguration() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argumentWithDynamicRefEntitySource = new Argument(); + argumentWithDynamicRefEntitySource.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + cfg.setArguments(Map.of("argumentWithDynamicRefEntitySource", argumentWithDynamicRefEntitySource)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } + + @Test + void validateShouldThrowWhenConfigurationHasNoArgumentsWithCurrentEntitySource() { var cfg = new PropagationCalculatedFieldConfiguration(); Argument argumentWithRefEntityIdSet = new Argument(); - argumentWithRefEntityIdSet.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("3703e895-3f9b-4b75-a715-b68f1ad51944"))); cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); + cfg.setApplyExpressionToResolvedArguments(true); assertThatThrownBy(cfg::validate) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + .hasMessage("At least one argument must be configured with the 'Current entity' " + + "source entity type for 'Expression result' propagation mode!"); } @Test From 8a3b8410ceccb0d77865ebd4996d13fc3d7a6c3b Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 16:55:56 +0300 Subject: [PATCH 037/122] Properly close CF state on removal --- ...CalculatedFieldEntityMessageProcessor.java | 33 ++++++++++++++----- ...alculatedFieldManagerMessageProcessor.java | 2 +- .../AbstractCalculatedFieldStateService.java | 2 +- .../cf/CalculatedFieldStateService.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 6 +++- .../cf/ctx/state/CalculatedFieldState.java | 6 +++- .../alarm/AlarmCalculatedFieldState.java | 23 +++++++------ 7 files changed, 51 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 3e9502bfe9..ccbdcc6b33 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -104,6 +104,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM "[{}][{}] Stopping entity actor due to change partition event." : "[{}][{}] Stopping entity actor.", tenantId, entityId); + states.values().forEach(this::closeState); states.clear(); actorCtx.stop(actorCtx.getSelf()); } @@ -123,7 +124,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.setPartition(msg.getPartition()); states.put(cfId, state); } else { - states.remove(cfId); + removeState(cfId); } } @@ -141,7 +142,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var ctx = msg.getCtx(); CalculatedFieldState state; if (msg.getStateAction() == StateAction.RECREATE) { - states.remove(ctx.getCfId()); + removeState(ctx.getCfId()); state = null; } else { state = states.get(ctx.getCfId()); @@ -195,14 +196,14 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); } else { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); - states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + states.forEach((cfId, state) -> cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); actorCtx.stop(actorCtx.getSelf()); } } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var state = states.remove(cfId); + var state = removeState(cfId); if (state != null) { - cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } else { msg.getCallback().onSuccess(); } @@ -423,14 +424,30 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state.isSizeOk()) { cfStateService.persistState(ctxId, state, callback); } else { - removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); + deleteStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); } } } - private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { + private CalculatedFieldState removeState(CalculatedFieldId cfId) { + CalculatedFieldState state = states.remove(cfId); + closeState(state); + return state; + } + + private void closeState(CalculatedFieldState state) { + if (state != null) { + try { + state.close(); + } catch (Exception e) { + log.warn("[{}][{}] Failed to close CF state", tenantId, state.getEntityId(), e); + } + } + } + + private void deleteStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { // We remove the state, but remember that it is over-sized in a local map. - cfStateService.removeState(ctxId, new TbCallback() { + cfStateService.deleteState(ctxId, new TbCallback() { @Override public void onSuccess() { callback.onFailure(ex); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index f0e8aa7906..14713d79b2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -141,7 +141,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - cfStateService.removeState(msg.getId(), msg.getCallback()); + cfStateService.deleteState(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index c8b99afd9e..e673577742 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -62,7 +62,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); @Override - public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + public final void deleteState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { doRemove(stateId, callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index d0b34f18e8..10276ac421 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -33,7 +33,7 @@ public interface CalculatedFieldStateService { void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; - void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + void deleteState(CalculatedFieldEntityCtxId stateId, TbCallback callback); void restore(QueueKey queueKey, Set partitions); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 153b83f8d4..e442964280 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -23,13 +23,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; +import java.io.Closeable; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @Getter -public abstract class BaseCalculatedFieldState implements CalculatedFieldState { +public abstract class BaseCalculatedFieldState implements CalculatedFieldState, Closeable { protected final EntityId entityId; protected CalculatedFieldCtx ctx; @@ -117,6 +118,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } } + @Override + public void close() {} + protected void validateNewEntry(String key, ArgumentEntry newEntry) {} private void updateLastUpdateTimestamp(ArgumentEntry entry) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index ff94206220..cc7188e7a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -29,6 +30,7 @@ import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldSta import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; +import java.io.Closeable; import java.util.Map; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -40,11 +42,13 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldState.class, name = "ALARM") }) -public interface CalculatedFieldState { +public interface CalculatedFieldState extends Closeable { @JsonIgnore CalculatedFieldType getType(); + EntityId getEntityId(); + Map getArguments(); long getLatestTimestamp(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index bc9ed4e32e..90999e5bbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -90,6 +90,8 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { private Alarm currentAlarm; private boolean initialFetchDone; + // TODO: deprecate device profile node, describe the differences and improvements + public AlarmCalculatedFieldState(EntityId entityId) { super(entityId); } @@ -107,7 +109,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public void init() { // todo: properly close state! + public void init() { super.init(); AtomicBoolean reevalNeeded = new AtomicBoolean(false); Map createRules = configuration.getCreateRules(); @@ -143,7 +145,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { AlarmEvalResult evalResult = state.reeval(System.currentTimeMillis()); if (evalResult.getStatus() == TRUE || evalResult.getStatus() == NOT_YET_TRUE) { ScheduledFuture future = ctx.scheduleReevaluation(evalResult.getLeftDuration(), actorCtx); - // TODO: use single task for multiple durations if durations are close enough. but be careful when cancelling the task in one of the states if (future != null) { state.setDurationCheckFuture(future); } @@ -167,17 +168,21 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { return ruleState; } - @Override - public Map update(Map argumentValues, CalculatedFieldCtx ctx) { - return super.update(argumentValues, ctx); - } - @Override public void reset() { super.reset(); configuration = null; } + @Override + public void close() { + super.close(); + for (AlarmRuleState state : createRuleStates.values()) { + clearState(state); + } + clearState(clearRuleState); + } + @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { initCurrentAlarm(ctx); @@ -186,10 +191,8 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { boolean newEvent = !updatedArgs.isEmpty(); AlarmEvalResult evalResult = state.eval(newEvent, ctx); if (evalResult.getStatus() == NOT_YET_TRUE && evalResult.getLeftDuration() > 0) { - // rounding up to the closest second -// long leftDuration = (long) Math.ceil(evalResult.getLeftDuration() / 1000.0) * 1000; long leftDuration = evalResult.getLeftDuration(); - ScheduledFuture future = ctx.scheduleReevaluation(leftDuration, actorCtx); // TODO: use single task for multiple durations if durations are close enough. but be careful when cancelling the task in one of the states + ScheduledFuture future = ctx.scheduleReevaluation(leftDuration, actorCtx); if (future != null) { state.setDurationCheckFuture(future); } From 3ce5215022260d316011d3a653b95ddd0c0c483c Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 10 Oct 2025 12:03:40 +0300 Subject: [PATCH 038/122] Alarm rules CF: reevaluate rule on schedule start --- .../server/actors/ActorSystemContext.java | 4 ++ ...alculatedFieldManagerMessageProcessor.java | 25 +++++++ .../cf/ctx/state/CalculatedFieldCtx.java | 2 + .../alarm/AlarmCalculatedFieldState.java | 9 ++- .../cf/ctx/state/alarm/AlarmRuleState.java | 64 +++++++++++------- .../src/main/resources/thingsboard.yml | 3 + .../thingsboard/server/cf/AlarmRulesTest.java | 66 +++++++++++++++++-- .../common/data/alarm/rule/AlarmRule.java | 4 ++ .../alarm/rule/condition/AlarmCondition.java | 5 ++ .../AlarmCalculatedFieldConfiguration.java | 6 ++ .../CalculatedFieldConfiguration.java | 4 ++ 11 files changed, 162 insertions(+), 30 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index f3e636296a..bf84163a8b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -664,6 +664,10 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; + @Value("${actors.alarms.reevaluation_interval:120}") + @Getter + private long alarmRulesReevaluationInterval; + @Autowired @Getter private MqttClientSettings mqttClientSettings; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 14713d79b2..4675821a5b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -66,6 +66,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -80,6 +82,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; @@ -122,6 +125,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.clear(); entityIdCalculatedFields.clear(); entityIdCalculatedFieldLinks.clear(); + if (cfsReevaluationTask != null) { + cfsReevaluationTask.cancel(true); + cfsReevaluationTask = null; + } ctx.stop(ctx.getSelf()); } @@ -129,6 +136,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); initEntitiesCache(); initCalculatedFields(); + scheduleCfsReevaluation(); msg.getCallback().onSuccess(); } @@ -149,6 +157,23 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware ctx.broadcastToChildren(msg, true); } + private void scheduleCfsReevaluation() { + cfsReevaluationTask = systemContext.getScheduler().scheduleWithFixedDelay(() -> { + try { + calculatedFields.values().forEach(cf -> { + if (cf.isRequiresScheduledReevaluation()) { + applyToTargetCfEntityActors(cf, TbCallback.EMPTY, (entityId, callback) -> { + log.debug("[{}][{}] Pushing scheduled CF reevaluate msg", entityId, cf.getCfId()); + getOrCreateActor(entityId).tell(new CalculatedFieldReevaluateMsg(tenantId, cf)); + }); + } + }); + } catch (Exception e) { + log.warn("[{}] Failed to trigger CFs reevaluation", tenantId, e); + } + }, systemContext.getAlarmRulesReevaluationInterval(), systemContext.getAlarmRulesReevaluationInterval(), TimeUnit.SECONDS); + } + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 9e9842c28d..e08abb8b50 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -85,6 +85,7 @@ public class CalculatedFieldCtx { private Output output; private String expression; private boolean useLatestTs; + private boolean requiresScheduledReevaluation; private ActorSystemContext systemContext; private TbelInvokeService tbelInvokeService; @@ -163,6 +164,7 @@ public class CalculatedFieldCtx { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } + this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 90999e5bbd..657fa80f63 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmCondition; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionType; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; @@ -142,7 +143,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { initCurrentAlarm(ctx); createOrClearAlarms(state -> { if (state.getCondition().getType() == AlarmConditionType.DURATION) { - AlarmEvalResult evalResult = state.reeval(System.currentTimeMillis()); + AlarmEvalResult evalResult = state.reeval(System.currentTimeMillis(), ctx); if (evalResult.getStatus() == TRUE || evalResult.getStatus() == NOT_YET_TRUE) { ScheduledFuture future = ctx.scheduleReevaluation(evalResult.getLeftDuration(), actorCtx); if (future != null) { @@ -161,7 +162,9 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } else { // when restored ruleState.setAlarmRule(rule); - if (rule.getCondition().getType() == AlarmConditionType.DURATION && !ruleState.isEmpty()) { + ruleState.setActive(null); + AlarmCondition condition = rule.getCondition(); + if (condition.hasSchedule() || (condition.getType() == AlarmConditionType.DURATION && !ruleState.isEmpty())) { reevalNeeded.set(true); } } @@ -199,7 +202,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } return evalResult; } else { - return state.reeval(System.currentTimeMillis()); + return state.reeval(System.currentTimeMillis(), ctx); } }, ctx); return Futures.immediateFuture(AlarmCalculatedFieldResult.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 97bf5c5453..9c1b966878 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -58,6 +58,7 @@ public class AlarmRuleState { private long lastEventTs; private transient long duration; private ScheduledFuture durationCheckFuture; + private Boolean active; public AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, AlarmCalculatedFieldState state) { this.severity = severity; @@ -68,32 +69,42 @@ public class AlarmRuleState { } public AlarmEvalResult eval(boolean newEvent, CalculatedFieldCtx ctx) { // on event or config change - boolean active = isActive(state.getLatestTimestamp()); - return switch (condition.getType()) { - case SIMPLE -> evalSimple(active, ctx); - case DURATION -> evalDuration(active, ctx); - case REPEATING -> evalRepeating(active, newEvent, ctx); - }; + long ts = newEvent ? state.getLatestTimestamp() : System.currentTimeMillis(); + active = isActive(ts); + if (!active) { + return AlarmEvalResult.FALSE; + } + return doEval(newEvent, ctx); } - public AlarmEvalResult reeval(long ts) { + public AlarmEvalResult reeval(long ts, CalculatedFieldCtx ctx) { // on scheduled duration check or periodic re-eval for rules with schedule + boolean active = isActive(ts); switch (condition.getType()) { case SIMPLE, REPEATING -> { - return AlarmEvalResult.NOT_YET_TRUE; + if (this.active == null || active != this.active) { + this.active = active; + if (active) { + return doEval(false, ctx); + } + } + if (active) { + return AlarmEvalResult.NOT_YET_TRUE; + } else { + return AlarmEvalResult.FALSE; + } } case DURATION -> { + if (!active) { + return AlarmEvalResult.FALSE; + } long requiredDuration = getRequiredDurationInMs(); if (requiredDuration > 0 && lastEventTs > 0 && ts > lastEventTs) { duration = ts - firstEventTs; - if (isActive(ts)) { - long leftDuration = requiredDuration - duration; - if (leftDuration <= 0) { - return AlarmEvalResult.TRUE; - } else { - return AlarmEvalResult.notYetTrue(0, leftDuration); - } + long leftDuration = requiredDuration - duration; + if (leftDuration <= 0) { + return AlarmEvalResult.TRUE; } else { - return AlarmEvalResult.FALSE; + return AlarmEvalResult.notYetTrue(0, leftDuration); } } } @@ -101,13 +112,20 @@ public class AlarmRuleState { return AlarmEvalResult.FALSE; } - private AlarmEvalResult evalSimple(boolean active, CalculatedFieldCtx ctx) { - return (active && eval(condition.getExpression(), ctx)) ? - AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; + public AlarmEvalResult doEval(boolean newEvent, CalculatedFieldCtx ctx) { + return switch (condition.getType()) { + case SIMPLE -> evalSimple(ctx); + case DURATION -> evalDuration(ctx); + case REPEATING -> evalRepeating(newEvent, ctx); + }; + } + + private AlarmEvalResult evalSimple(CalculatedFieldCtx ctx) { + return eval(condition.getExpression(), ctx) ? AlarmEvalResult.TRUE : AlarmEvalResult.FALSE; } - private AlarmEvalResult evalRepeating(boolean active, boolean newEvent, CalculatedFieldCtx ctx) { - if (active && eval(condition.getExpression(), ctx)) { + private AlarmEvalResult evalRepeating(boolean newEvent, CalculatedFieldCtx ctx) { + if (eval(condition.getExpression(), ctx)) { if (newEvent) { eventCount++; } @@ -123,8 +141,8 @@ public class AlarmRuleState { } } - private AlarmEvalResult evalDuration(boolean active, CalculatedFieldCtx ctx) { - if (active && eval(condition.getExpression(), ctx)) { + private AlarmEvalResult evalDuration(CalculatedFieldCtx ctx) { + if (eval(condition.getExpression(), ctx)) { long eventTs = state.getLatestTimestamp(); if (lastEventTs > 0) { if (eventTs > lastEventTs) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2f3b40bf12..df4544e898 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -529,6 +529,9 @@ actors: configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" + alarms: + # Interval in seconds to re-evaluate Alarm rules that have a time schedule. 2 minutes by default. + reevaluation_interval: "${ACTORS_ALARMS_REEVALUATION_INTERVAL_SEC:120}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index f468bd709b..020ad0c60d 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.action.TbAlarmResult; @@ -45,6 +46,7 @@ import org.thingsboard.server.common.data.alarm.rule.condition.expression.predic import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.StringFilterPredicate.StringOperation; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.SpecificTimeSchedule; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; @@ -63,18 +65,27 @@ import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Predicate; import static org.assertj.core.api.Assertions.assertThat; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; @Slf4j @DaoSqlTest +@TestPropertySource(properties = { + "actors.alarms.reevaluation_interval=1" +}) public class AlarmRulesTest extends AbstractControllerTest { @MockitoSpyBean @@ -209,10 +220,15 @@ public class AlarmRulesTest extends AbstractControllerTest { assertThat(alarmResult.getConditionRepeats()).isEqualTo(5); }); - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 4; i++) { postTelemetry(deviceId, "{\"temperature\":50}"); Thread.sleep(10); } + checkAlarmResult(calculatedField, alarmResult -> alarmResult.getConditionRepeats() == 9, alarmResult -> { + assertThat(alarmResult.isUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.MAJOR); + }); + postTelemetry(deviceId, "{\"temperature\":50}"); checkAlarmResult(calculatedField, alarmResult -> { assertThat(alarmResult.isSeverityUpdated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); @@ -420,7 +436,7 @@ public class AlarmRulesTest extends AbstractControllerTest { scheduleArgument.setDefaultValue("None"); Map arguments = Map.of( "temperature", temperatureArgument, - "schedule", scheduleArgument // fixme: + "schedule", scheduleArgument ); Map createRules = Map.of( @@ -638,11 +654,53 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + @Test + public void testCreateAlarm_scheduleStarted() throws Exception { + Argument parkingSpotOccupiedArgument = new Argument(); + parkingSpotOccupiedArgument.setRefEntityKey(new ReferencedEntityKey("parkingSpotOccupied", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + parkingSpotOccupiedArgument.setDefaultValue("false"); + Map arguments = Map.of( + "parkingSpotOccupied", parkingSpotOccupiedArgument + ); + + SpecificTimeSchedule schedule = new SpecificTimeSchedule(); + schedule.setTimezone(ZoneId.systemDefault().getId()); + schedule.setDaysOfWeek(Set.of(1, 2, 3, 4, 5, 6, 7)); + long startsOn = Duration.between(LocalDate.now().atStartOfDay(), LocalDateTime.now()) + .plus(15, ChronoUnit.SECONDS).toMillis(); + schedule.setStartsOn(startsOn); + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return parkingSpotOccupied == true;", null, null, null, + new AlarmConditionValue<>(schedule, null)) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "Illegal parking alarm", + arguments, createRules, null); + + postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"parkingSpotOccupied\":true}"); + + Thread.sleep(10000); + assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); + + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + // TODO: MSA tests private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { + checkAlarmResult(calculatedField, null, assertion); + } + + private void checkAlarmResult(CalculatedField calculatedField, + Predicate waitFor, + Consumer assertion) { TbAlarmResult alarmResult = await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> getLatestAlarmResult(calculatedField.getId()), Objects::nonNull); + .until(() -> getLatestAlarmResult(calculatedField.getId()), result -> + result != null && (waitFor == null || waitFor.test(result))); assertion.accept(alarmResult); Alarm alarm = alarmResult.getAlarm(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java index bd4c4b0dd8..ab7adcbd48 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java @@ -30,4 +30,8 @@ public class AlarmRule { private String alarmDetails; private DashboardId dashboardId; + public boolean requiresScheduledReevaluation() { + return condition.hasSchedule(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java index a13de08480..c9280151d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java @@ -26,6 +26,7 @@ import lombok.NoArgsConstructor; import org.jetbrains.annotations.NotNull; import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionExpression; import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AlarmSchedule; +import org.thingsboard.server.common.data.alarm.rule.condition.schedule.AnyTimeSchedule; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @@ -44,6 +45,10 @@ public abstract class AlarmCondition { @Valid private AlarmConditionValue schedule; + public boolean hasSchedule() { + return schedule != null && !(schedule.getStaticValue() instanceof AnyTimeSchedule); + } + @JsonIgnore public abstract AlarmConditionType getType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java index 0b0f34ad50..9b40a77cbe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -58,4 +58,10 @@ public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculat } + @Override + public boolean requiresScheduledReevaluation() { + return createRules.values().stream().anyMatch(AlarmRule::requiresScheduledReevaluation) || + (clearRule != null && clearRule.requiresScheduledReevaluation()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 7b608192db..d3622a2dcf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -72,4 +72,8 @@ public interface CalculatedFieldConfiguration { .collect(Collectors.toList()); } + default boolean requiresScheduledReevaluation() { + return false; + } + } From 8f1b2b832f20f110f2217e68b7e0fb641d52b2c7 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 10 Oct 2025 16:20:34 +0300 Subject: [PATCH 039/122] Deprecate device profile node --- .../rule_chains/edge_root_rule_chain.json | 125 +++----- .../device_profile/rule_chain_template.json | 44 +-- .../tenant/rule_chains/root_rule_chain.json | 20 +- .../server/actors/app/AppActor.java | 2 +- .../DefaultSystemDataLoaderService.java | 279 ++++++++++-------- .../processing/AbstractConsumerService.java | 1 + .../thingsboard/server/cf/AlarmRulesTest.java | 1 + .../SimpleAlarmConditionExpression.java | 4 + .../data/device/profile/AlarmCondition.java | 1 + .../device/profile/AlarmConditionFilter.java | 1 + .../profile/AlarmConditionFilterKey.java | 1 + .../device/profile/AlarmConditionKeyType.java | 1 + .../device/profile/AlarmConditionSpec.java | 1 + .../profile/AlarmConditionSpecType.java | 1 + .../common/data/device/profile/AlarmRule.java | 1 + .../data/device/profile/AlarmSchedule.java | 1 + .../device/profile/AlarmScheduleType.java | 1 + .../data/device/profile/AnyTimeSchedule.java | 1 + .../device/profile/CustomTimeSchedule.java | 1 + .../profile/CustomTimeScheduleItem.java | 1 + .../device/profile/DeviceProfileAlarm.java | 1 + .../profile/DurationAlarmConditionSpec.java | 1 + .../profile/RepeatingAlarmConditionSpec.java | 1 + .../profile/SimpleAlarmConditionSpec.java | 1 + .../device/profile/SpecificTimeSchedule.java | 1 + .../server/common/data/msg/TbMsgType.java | 2 +- .../rule/engine/profile/AlarmEvalResult.java | 1 + .../rule/engine/profile/AlarmRuleState.java | 1 + .../rule/engine/profile/AlarmState.java | 1 + .../rule/engine/profile/DataSnapshot.java | 1 + .../rule/engine/profile/DeviceState.java | 1 + .../profile/DynamicPredicateValueCtx.java | 1 + .../profile/DynamicPredicateValueCtxImpl.java | 1 + .../rule/engine/profile/EntityKeyValue.java | 1 + .../rule/engine/profile/ProfileState.java | 1 + .../rule/engine/profile/SnapshotUpdate.java | 1 + .../engine/profile/TbDeviceProfileNode.java | 5 +- .../TbDeviceProfileNodeConfiguration.java | 1 + .../state/PersistedAlarmRuleState.java | 1 + .../profile/state/PersistedAlarmState.java | 1 + .../profile/state/PersistedDeviceState.java | 1 + 41 files changed, 258 insertions(+), 256 deletions(-) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index 81f9e6a14d..e614c9b54c 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -10,27 +10,9 @@ "externalId": null }, "metadata": { - "firstNodeIndex": 0, + "firstNodeIndex": 2, "nodes": [ { - "additionalInfo": { - "description": "Process incoming messages from devices with the alarm rules defined in the device profile. Dispatch all incoming messages with \"Success\" relation type.", - "layoutX": 187, - "layoutY": 468 - }, - "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", - "name": "Device Profile Node", - "configuration": { - "persistAlarmRulesState": false, - "fetchAlarmRulesStateOnStart": false - }, - "externalId": null - }, - { - "additionalInfo": { - "layoutX": 823, - "layoutY": 157 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configurationVersion": 1, @@ -41,13 +23,12 @@ "type": "ON_EVERY_MESSAGE" } }, - "externalId": null + "additionalInfo": { + "layoutX": 823, + "layoutY": 157 + } }, { - "additionalInfo": { - "layoutX": 824, - "layoutY": 52 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Client Attributes", "configurationVersion": 3, @@ -60,25 +41,23 @@ "sendAttributesUpdatedNotification": false, "updateAttributesOnlyOnValueChange": true }, - "externalId": null + "additionalInfo": { + "layoutX": 824, + "layoutY": 52 + } }, { - "additionalInfo": { - "layoutX": 347, - "layoutY": 149 - }, "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", "name": "Message Type Switch", "configuration": { "version": 0 }, - "externalId": null + "additionalInfo": { + "layoutX": 347, + "layoutY": 149 + } }, { - "additionalInfo": { - "layoutX": 825, - "layoutY": 266 - }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log RPC from Device", "configuration": { @@ -86,13 +65,12 @@ "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" }, - "externalId": null + "additionalInfo": { + "layoutX": 825, + "layoutY": 266 + } }, { - "additionalInfo": { - "layoutX": 824, - "layoutY": 378 - }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log Other", "configuration": { @@ -100,97 +78,92 @@ "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" }, - "externalId": null - }, - { "additionalInfo": { "layoutX": 824, - "layoutY": 466 - }, + "layoutY": 378 + } + }, + { "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", "name": "RPC Call Request", "configuration": { "timeoutInSeconds": 60 }, - "externalId": null + "additionalInfo": { + "layoutX": 824, + "layoutY": 466 + } }, { - "additionalInfo": { - "layoutX": 1126, - "layoutY": 104 - }, "type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode", "name": "Push to cloud", "configuration": { "scope": "CLIENT_SCOPE" }, - "externalId": null + "additionalInfo": { + "layoutX": 1126, + "layoutY": 104 + } }, { - "additionalInfo": { - "layoutX": 826, - "layoutY": 601 - }, "type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode", "name": "Push to cloud", "configuration": { "scope": "SERVER_SCOPE" }, - "externalId": null + "additionalInfo": { + "layoutX": 826, + "layoutY": 601 + } } ], "connections": [ { "fromIndex": 0, - "toIndex": 3, + "toIndex": 6, "type": "Success" }, { "fromIndex": 1, - "toIndex": 7, + "toIndex": 6, "type": "Success" }, { "fromIndex": 2, - "toIndex": 7, - "type": "Success" - }, - { - "fromIndex": 3, - "toIndex": 1, + "toIndex": 0, "type": "Post telemetry" }, { - "fromIndex": 3, - "toIndex": 2, + "fromIndex": 2, + "toIndex": 1, "type": "Post attributes" }, { - "fromIndex": 3, - "toIndex": 4, + "fromIndex": 2, + "toIndex": 3, "type": "RPC Request from Device" }, { - "fromIndex": 3, - "toIndex": 5, + "fromIndex": 2, + "toIndex": 4, "type": "Other" }, { - "fromIndex": 3, - "toIndex": 6, + "fromIndex": 2, + "toIndex": 5, "type": "RPC Request to Device" }, { - "fromIndex": 3, - "toIndex": 8, + "fromIndex": 2, + "toIndex": 7, "type": "Attributes Deleted" }, { - "fromIndex": 3, - "toIndex": 8, + "fromIndex": 2, + "toIndex": 7, "type": "Attributes Updated" } ], "ruleChainConnections": null } -} +} \ No newline at end of file diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 305dc04961..8773a2d6aa 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -10,12 +10,12 @@ "configuration": null }, "metadata": { - "firstNodeIndex": 6, + "firstNodeIndex": 2, "nodes": [ { "additionalInfo": { - "layoutX": 822, - "layoutY": 294 + "layoutX": 824, + "layoutY": 156 }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", @@ -30,8 +30,8 @@ }, { "additionalInfo": { - "layoutX": 824, - "layoutY": 221 + "layoutX": 825, + "layoutY": 52 }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Client Attributes", @@ -48,8 +48,8 @@ }, { "additionalInfo": { - "layoutX": 494, - "layoutY": 309 + "layoutX": 347, + "layoutY": 149 }, "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", "name": "Message Type Switch", @@ -59,8 +59,8 @@ }, { "additionalInfo": { - "layoutX": 824, - "layoutY": 383 + "layoutX": 825, + "layoutY": 266 }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log RPC from Device", @@ -72,8 +72,8 @@ }, { "additionalInfo": { - "layoutX": 823, - "layoutY": 444 + "layoutX": 825, + "layoutY": 379 }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log Other", @@ -85,27 +85,14 @@ }, { "additionalInfo": { - "layoutX": 822, - "layoutY": 507 + "layoutX": 825, + "layoutY": 468 }, "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", "name": "RPC Call Request", "configuration": { "timeoutInSeconds": 60 } - }, - { - "additionalInfo": { - "description": "", - "layoutX": 209, - "layoutY": 307 - }, - "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", - "name": "Device Profile Node", - "configuration": { - "persistAlarmRulesState": false, - "fetchAlarmRulesStateOnStart": false - } } ], "connections": [ @@ -133,11 +120,6 @@ "fromIndex": 2, "toIndex": 5, "type": "RPC Request to Device" - }, - { - "fromIndex": 6, - "toIndex": 2, - "type": "Success" } ], "ruleChainConnections": null diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index a988c9d5eb..c48dab1964 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -9,7 +9,7 @@ "configuration": null }, "metadata": { - "firstNodeIndex": 6, + "firstNodeIndex": 2, "nodes": [ { "additionalInfo": { @@ -92,27 +92,9 @@ "configuration": { "timeoutInSeconds": 60 } - }, - { - "additionalInfo": { - "description": "Process incoming messages from devices with the alarm rules defined in the device profile. Dispatch all incoming messages with \"Success\" relation type.", - "layoutX": 204, - "layoutY": 240 - }, - "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", - "name": "Device Profile Node", - "configuration": { - "persistAlarmRulesState": false, - "fetchAlarmRulesStateOnStart": false - } } ], "connections": [ - { - "fromIndex": 6, - "toIndex": 2, - "type": "Success" - }, { "fromIndex": 2, "toIndex": 4, diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 4715ea64d4..20cacda26a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -165,7 +165,7 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { - if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) { + if (!msg.getEntityId().getEntityType().isOneOf(EntityType.TENANT_PROFILE, EntityType.TB_RESOURCE)) { log.warn("Message has system tenant id: {}", msg); } } else { diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index d580175aa0..287581d297 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -49,17 +49,25 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.device.profile.AlarmCondition; -import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; -import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; -import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; -import org.thingsboard.server.common.data.device.profile.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; +import org.thingsboard.server.common.data.alarm.rule.condition.SimpleAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.AlarmConditionFilter; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.ComplexOperation; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.SimpleAlarmConditionExpression; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.BooleanFilterPredicate; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.NumericFilterPredicate; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfileProvisionConfiguration; -import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpec; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -74,12 +82,6 @@ import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.query.BooleanFilterPredicate; -import org.thingsboard.server.common.data.query.DynamicValue; -import org.thingsboard.server.common.data.query.DynamicValueSourceType; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.queue.ProcessingStrategy; import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; @@ -94,6 +96,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -117,7 +120,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.List; -import java.util.TreeMap; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -155,6 +158,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private final MobileAppDao mobileAppDao; private final NotificationSettingsService notificationSettingsService; private final NotificationTargetService notificationTargetService; + private final CalculatedFieldService calculatedFieldService; @Autowired private BCryptPasswordEncoder passwordEncoder; @@ -306,8 +310,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { if (invalidSignKey) { log.warn("WARNING: {}. A new JWT Signing Key has been added automatically. " + - "You can change the JWT Signing Key using the Web UI: " + - "Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.", warningMessage); + "You can change the JWT Signing Key using the Web UI: " + + "Navigate to \"System settings -> Security settings\" while logged in as a System Administrator.", warningMessage); jwtSettings.setTokenSigningKey(generateRandomKey()); jwtSettingsService.saveJwtSettings(jwtSettings); @@ -319,9 +323,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) .forEach(mobileApp -> { log.warn("WARNING: The App secret is shorter than 512 bits, which is a security risk. " + - "A new Application Secret has been added automatically for Mobile Application [{}]. " + - "You can change the Application Secret using the Web UI: " + - "Navigate to \"Security settings -> OAuth2 -> Mobile applications\" while logged in as a System Administrator.", mobileApp.getPkgName()); + "A new Application Secret has been added automatically for Mobile Application [{}]. " + + "You can change the Application Secret using the Web UI: " + + "Navigate to \"Security settings -> OAuth2 -> Mobile applications\" while logged in as a System Administrator.", mobileApp.getPkgName()); mobileApp.setAppSecret(generateRandomKey()); mobileAppDao.save(TenantId.SYS_TENANT_ID, mobileApp); }); @@ -372,11 +376,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), customerB.getId(), defaultDeviceProfile.getId(), "Test Device B1", "B1_TEST_TOKEN", null); createDevice(demoTenant.getId(), customerC.getId(), defaultDeviceProfile.getId(), "Test Device C1", "C1_TEST_TOKEN", null); - createDevice(demoTenant.getId(), null, defaultDeviceProfile.getId(), "DHT11 Demo Device", "DHT11_DEMO_TOKEN", "Demo device that is used in sample " + - "applications that upload data from DHT11 temperature and humidity sensor"); + createDevice(demoTenant.getId(), null, defaultDeviceProfile.getId(), "DHT11 Demo Device", "DHT11_DEMO_TOKEN", + "Demo device that is used in sample applications that upload data from DHT11 temperature and humidity sensor"); - createDevice(demoTenant.getId(), null, defaultDeviceProfile.getId(), "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + - "Raspberry Pi GPIO control sample application"); + createDevice(demoTenant.getId(), null, defaultDeviceProfile.getId(), "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", + "Demo device that is used in Raspberry Pi GPIO control sample application"); DeviceProfile thermostatDeviceProfile = new DeviceProfile(); thermostatDeviceProfile.setTenantId(demoTenant.getId()); @@ -398,110 +402,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { deviceProfileData.setProvisionConfiguration(provisionConfiguration); thermostatDeviceProfile.setProfileData(deviceProfileData); - DeviceProfileAlarm highTemperature = new DeviceProfileAlarm(); - highTemperature.setId("highTemperatureAlarmID"); - highTemperature.setAlarmType("High Temperature"); - AlarmRule temperatureRule = new AlarmRule(); - AlarmCondition temperatureCondition = new AlarmCondition(); - temperatureCondition.setSpec(new SimpleAlarmConditionSpec()); - - AlarmConditionFilter temperatureAlarmFlagAttributeFilter = new AlarmConditionFilter(); - temperatureAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "temperatureAlarmFlag")); - temperatureAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); - BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate(); - temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); - temperatureAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); - temperatureAlarmFlagAttributeFilter.setPredicate(temperatureAlarmFlagAttributePredicate); - - AlarmConditionFilter temperatureTimeseriesFilter = new AlarmConditionFilter(); - temperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - temperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate temperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); - temperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); - FilterPredicateValue temperatureTimeseriesPredicateValue = - new FilterPredicateValue<>(25.0, null, - new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "temperatureAlarmThreshold")); - temperatureTimeseriesFilterPredicate.setValue(temperatureTimeseriesPredicateValue); - temperatureTimeseriesFilter.setPredicate(temperatureTimeseriesFilterPredicate); - temperatureCondition.setCondition(Arrays.asList(temperatureAlarmFlagAttributeFilter, temperatureTimeseriesFilter)); - temperatureRule.setAlarmDetails("Current temperature = ${temperature}"); - temperatureRule.setCondition(temperatureCondition); - highTemperature.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.MAJOR, temperatureRule))); - - AlarmRule clearTemperatureRule = new AlarmRule(); - AlarmCondition clearTemperatureCondition = new AlarmCondition(); - clearTemperatureCondition.setSpec(new SimpleAlarmConditionSpec()); - - AlarmConditionFilter clearTemperatureTimeseriesFilter = new AlarmConditionFilter(); - clearTemperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - clearTemperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate clearTemperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); - clearTemperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL); - FilterPredicateValue clearTemperatureTimeseriesPredicateValue = - new FilterPredicateValue<>(25.0, null, - new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "temperatureAlarmThreshold")); - - clearTemperatureTimeseriesFilterPredicate.setValue(clearTemperatureTimeseriesPredicateValue); - clearTemperatureTimeseriesFilter.setPredicate(clearTemperatureTimeseriesFilterPredicate); - clearTemperatureCondition.setCondition(Collections.singletonList(clearTemperatureTimeseriesFilter)); - clearTemperatureRule.setCondition(clearTemperatureCondition); - clearTemperatureRule.setAlarmDetails("Current temperature = ${temperature}"); - highTemperature.setClearRule(clearTemperatureRule); - - DeviceProfileAlarm lowHumidity = new DeviceProfileAlarm(); - lowHumidity.setId("lowHumidityAlarmID"); - lowHumidity.setAlarmType("Low Humidity"); - AlarmRule humidityRule = new AlarmRule(); - AlarmCondition humidityCondition = new AlarmCondition(); - humidityCondition.setSpec(new SimpleAlarmConditionSpec()); - - AlarmConditionFilter humidityAlarmFlagAttributeFilter = new AlarmConditionFilter(); - humidityAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "humidityAlarmFlag")); - humidityAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); - BooleanFilterPredicate humidityAlarmFlagAttributePredicate = new BooleanFilterPredicate(); - humidityAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); - humidityAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); - humidityAlarmFlagAttributeFilter.setPredicate(humidityAlarmFlagAttributePredicate); - - AlarmConditionFilter humidityTimeseriesFilter = new AlarmConditionFilter(); - humidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity")); - humidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate humidityTimeseriesFilterPredicate = new NumericFilterPredicate(); - humidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); - FilterPredicateValue humidityTimeseriesPredicateValue = - new FilterPredicateValue<>(60.0, null, - new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "humidityAlarmThreshold")); - humidityTimeseriesFilterPredicate.setValue(humidityTimeseriesPredicateValue); - humidityTimeseriesFilter.setPredicate(humidityTimeseriesFilterPredicate); - humidityCondition.setCondition(Arrays.asList(humidityAlarmFlagAttributeFilter, humidityTimeseriesFilter)); - - humidityRule.setCondition(humidityCondition); - humidityRule.setAlarmDetails("Current humidity = ${humidity}"); - lowHumidity.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.MINOR, humidityRule))); - - AlarmRule clearHumidityRule = new AlarmRule(); - AlarmCondition clearHumidityCondition = new AlarmCondition(); - clearHumidityCondition.setSpec(new SimpleAlarmConditionSpec()); - - AlarmConditionFilter clearHumidityTimeseriesFilter = new AlarmConditionFilter(); - clearHumidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity")); - clearHumidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate clearHumidityTimeseriesFilterPredicate = new NumericFilterPredicate(); - clearHumidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL); - FilterPredicateValue clearHumidityTimeseriesPredicateValue = - new FilterPredicateValue<>(60.0, null, - new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "humidityAlarmThreshold")); - - clearHumidityTimeseriesFilterPredicate.setValue(clearHumidityTimeseriesPredicateValue); - clearHumidityTimeseriesFilter.setPredicate(clearHumidityTimeseriesFilterPredicate); - clearHumidityCondition.setCondition(Collections.singletonList(clearHumidityTimeseriesFilter)); - clearHumidityRule.setCondition(clearHumidityCondition); - clearHumidityRule.setAlarmDetails("Current humidity = ${humidity}"); - lowHumidity.setClearRule(clearHumidityRule); - - deviceProfileData.setAlarms(Arrays.asList(highTemperature, lowHumidity)); - DeviceProfile savedThermostatDeviceProfile = deviceProfileService.saveDeviceProfile(thermostatDeviceProfile); + createAlarmRules(demoTenant.getId(), savedThermostatDeviceProfile.getId()); DeviceId t1Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); DeviceId t2Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); @@ -526,6 +428,130 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { installScripts.createDefaultTenantDashboards(demoTenant.getId(), null); } + private void createAlarmRules(TenantId tenantId, DeviceProfileId deviceProfileId) { + CalculatedField highTemperature = new CalculatedField(); + highTemperature.setName("High Temperature"); + highTemperature.setType(CalculatedFieldType.ALARM); + highTemperature.setTenantId(tenantId); + highTemperature.setEntityId(deviceProfileId); + highTemperature.setDebugSettings(DebugSettings.all()); + AlarmCalculatedFieldConfiguration highTemperatureConfig = new AlarmCalculatedFieldConfiguration(); + highTemperature.setConfiguration(highTemperatureConfig); + + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + Argument temperatureThresholdArgument = new Argument(); + temperatureThresholdArgument.setRefEntityKey(new ReferencedEntityKey("temperatureAlarmThreshold", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + temperatureThresholdArgument.setDefaultValue("25"); + Argument temperatureAlarmFlagArgument = new Argument(); + temperatureAlarmFlagArgument.setRefEntityKey(new ReferencedEntityKey("temperatureAlarmFlag", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + highTemperatureConfig.setArguments(Map.of( + "temperature", temperatureArgument, + "temperatureAlarmThreshold", temperatureThresholdArgument, + "temperatureAlarmFlag", temperatureAlarmFlagArgument + )); + + AlarmRule temperatureRule = new AlarmRule(); + SimpleAlarmCondition temperatureCondition = new SimpleAlarmCondition(); + + AlarmConditionFilter temperatureAlarmFlagFilter = new AlarmConditionFilter(); + temperatureAlarmFlagFilter.setArgument("temperatureAlarmFlag"); + BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate(); + temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + temperatureAlarmFlagAttributePredicate.setValue(new AlarmConditionValue<>(Boolean.TRUE, null)); + temperatureAlarmFlagFilter.setPredicate(temperatureAlarmFlagAttributePredicate); + + AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); + temperatureFilter.setArgument("temperature"); + NumericFilterPredicate temperatureFilterPredicate = new NumericFilterPredicate(); + temperatureFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + temperatureFilterPredicate.setValue(new AlarmConditionValue<>(null, "temperatureAlarmThreshold")); + temperatureFilter.setPredicate(temperatureFilterPredicate); + temperatureCondition.setExpression(new SimpleAlarmConditionExpression(List.of(temperatureAlarmFlagFilter, temperatureFilter), ComplexOperation.AND)); + temperatureRule.setCondition(temperatureCondition); + temperatureRule.setAlarmDetails("Current temperature = ${temperature}"); + highTemperatureConfig.setCreateRules(Map.of( + AlarmSeverity.MAJOR, temperatureRule + )); + + AlarmRule clearTemperatureRule = new AlarmRule(); + SimpleAlarmCondition clearTemperatureCondition = new SimpleAlarmCondition(); + + AlarmConditionFilter clearTemperatureFilter = new AlarmConditionFilter(); + clearTemperatureFilter.setArgument("temperature"); + NumericFilterPredicate clearTemperatureFilterPredicate = new NumericFilterPredicate(); + clearTemperatureFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL); + clearTemperatureFilterPredicate.setValue(new AlarmConditionValue<>(null, "temperatureAlarmThreshold")); + clearTemperatureFilter.setPredicate(clearTemperatureFilterPredicate); + clearTemperatureCondition.setExpression(new SimpleAlarmConditionExpression(List.of(clearTemperatureFilter), ComplexOperation.AND)); + clearTemperatureRule.setCondition(clearTemperatureCondition); + clearTemperatureRule.setAlarmDetails("Current temperature = ${temperature}"); + highTemperatureConfig.setClearRule(clearTemperatureRule); + + calculatedFieldService.save(highTemperature); + + CalculatedField lowHumidity = new CalculatedField(); + lowHumidity.setName("Low Humidity"); + lowHumidity.setType(CalculatedFieldType.ALARM); + lowHumidity.setTenantId(tenantId); + lowHumidity.setEntityId(deviceProfileId); + lowHumidity.setDebugSettings(DebugSettings.all()); + AlarmCalculatedFieldConfiguration lowHumidityConfig = new AlarmCalculatedFieldConfiguration(); + lowHumidity.setConfiguration(lowHumidityConfig); + + Argument humidityArgument = new Argument(); + humidityArgument.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null)); + Argument humidityThresholdArgument = new Argument(); + humidityThresholdArgument.setRefEntityKey(new ReferencedEntityKey("humidityAlarmThreshold", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + humidityThresholdArgument.setDefaultValue("60"); + Argument humidityAlarmFlagArgument = new Argument(); + humidityAlarmFlagArgument.setRefEntityKey(new ReferencedEntityKey("humidityAlarmFlag", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + lowHumidityConfig.setArguments(Map.of( + "humidity", humidityArgument, + "humidityAlarmThreshold", humidityThresholdArgument, + "humidityAlarmFlag", humidityAlarmFlagArgument + )); + + AlarmRule humidityRule = new AlarmRule(); + SimpleAlarmCondition humidityCondition = new SimpleAlarmCondition(); + + AlarmConditionFilter humidityAlarmFlagAttributeFilter = new AlarmConditionFilter(); + humidityAlarmFlagAttributeFilter.setArgument("humidityAlarmFlag"); + BooleanFilterPredicate humidityAlarmFlagPredicate = new BooleanFilterPredicate(); + humidityAlarmFlagPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + humidityAlarmFlagPredicate.setValue(new AlarmConditionValue<>(Boolean.TRUE, null)); + humidityAlarmFlagAttributeFilter.setPredicate(humidityAlarmFlagPredicate); + + AlarmConditionFilter humidityFilter = new AlarmConditionFilter(); + humidityFilter.setArgument("humidity"); + NumericFilterPredicate humidityFilterPredicate = new NumericFilterPredicate(); + humidityFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); + humidityFilterPredicate.setValue(new AlarmConditionValue<>(null, "humidityAlarmThreshold")); + humidityFilter.setPredicate(humidityFilterPredicate); + humidityCondition.setExpression(new SimpleAlarmConditionExpression(List.of(humidityAlarmFlagAttributeFilter, humidityFilter), ComplexOperation.AND)); + humidityRule.setCondition(humidityCondition); + humidityRule.setAlarmDetails("Current humidity = ${humidity}"); + lowHumidityConfig.setCreateRules(Map.of( + AlarmSeverity.MINOR, humidityRule + )); + + AlarmRule clearHumidityRule = new AlarmRule(); + SimpleAlarmCondition clearHumidityCondition = new SimpleAlarmCondition(); + + AlarmConditionFilter clearHumidityFilter = new AlarmConditionFilter(); + clearHumidityFilter.setArgument("humidity"); + NumericFilterPredicate clearHumidityFilterPredicate = new NumericFilterPredicate(); + clearHumidityFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL); + clearHumidityFilterPredicate.setValue(new AlarmConditionValue<>(null, "humidityAlarmThreshold")); + clearHumidityFilter.setPredicate(clearHumidityFilterPredicate); + clearHumidityCondition.setExpression(new SimpleAlarmConditionExpression(List.of(clearHumidityFilter), ComplexOperation.AND)); + clearHumidityRule.setCondition(clearHumidityCondition); + clearHumidityRule.setAlarmDetails("Current humidity = ${humidity}"); + lowHumidityConfig.setClearRule(clearHumidityRule); + + calculatedFieldService.save(lowHumidity); + } + @Override public void loadSystemWidgets() throws Exception { installScripts.loadSystemWidgets(); @@ -609,6 +635,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { public void onFailure(Throwable t) { log.warn("[{}] Failed to update attribute [{}] with value [{}]", deviceId, key, value, t); } + } private void addTsCallback(ListenableFuture saveFuture, final FutureCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 7382ef1c4d..6e162256a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -228,6 +228,7 @@ public abstract class AbstractConsumerService assertion) { checkAlarmResult(calculatedField, null, assertion); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java index e541fbfd31..8c27400961 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/SimpleAlarmConditionExpression.java @@ -17,11 +17,15 @@ package org.thingsboard.server.common.data.alarm.rule.condition.expression; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; @Data +@AllArgsConstructor +@NoArgsConstructor public class SimpleAlarmConditionExpression implements AlarmConditionExpression { @Valid diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java index 07c42eb31b..f840886834 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java @@ -26,6 +26,7 @@ import java.util.List; @Schema @Data @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class AlarmCondition implements Serializable { @Valid diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java index 210193fc01..6e7e6ab321 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java @@ -26,6 +26,7 @@ import java.io.Serializable; @Schema @Data +@Deprecated public class AlarmConditionFilter implements Serializable { @Valid diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java index d31e6710ef..3c6e5252a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java @@ -23,6 +23,7 @@ import java.io.Serializable; @Schema @Data +@Deprecated public class AlarmConditionFilterKey implements Serializable { @Schema(description = "The key type", example = "TIME_SERIES") diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java index 9eef80e312..6f451a1abc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.device.profile; +@Deprecated public enum AlarmConditionKeyType { ATTRIBUTE, TIME_SERIES, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpec.java index 37b2a9d7c5..f3f969f641 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpec.java @@ -31,6 +31,7 @@ import java.io.Serializable; @JsonSubTypes.Type(value = SimpleAlarmConditionSpec.class, name = "SIMPLE"), @JsonSubTypes.Type(value = DurationAlarmConditionSpec.class, name = "DURATION"), @JsonSubTypes.Type(value = RepeatingAlarmConditionSpec.class, name = "REPEATING")}) +@Deprecated public interface AlarmConditionSpec extends Serializable { @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpecType.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpecType.java index adef445914..229be24b42 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpecType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionSpecType.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.device.profile; +@Deprecated public enum AlarmConditionSpecType { SIMPLE, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java index 16850e3669..64b516cd25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java @@ -25,6 +25,7 @@ import java.io.Serializable; @Schema @Data +@Deprecated public class AlarmRule implements Serializable { @Valid diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java index 09e8d3c146..4bfa2ef9f6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmSchedule.java @@ -31,6 +31,7 @@ import java.io.Serializable; @JsonSubTypes.Type(value = AnyTimeSchedule.class, name = "ANY_TIME"), @JsonSubTypes.Type(value = SpecificTimeSchedule.class, name = "SPECIFIC_TIME"), @JsonSubTypes.Type(value = CustomTimeSchedule.class, name = "CUSTOM")}) +@Deprecated public interface AlarmSchedule extends Serializable { AlarmScheduleType getType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmScheduleType.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmScheduleType.java index f50a3b47db..ab06cb9335 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmScheduleType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmScheduleType.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.device.profile; +@Deprecated public enum AlarmScheduleType { ANY_TIME, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java index 426430481a..87766d685c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AnyTimeSchedule.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.device.profile; import org.thingsboard.server.common.data.query.DynamicValue; +@Deprecated public class AnyTimeSchedule implements AlarmSchedule { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java index b372a2fa07..6b07341b30 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeSchedule.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.query.DynamicValue; import java.util.List; @Data +@Deprecated public class CustomTimeSchedule implements AlarmSchedule { private String timezone; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeScheduleItem.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeScheduleItem.java index abcbec4e32..b0781e3ad1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeScheduleItem.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/CustomTimeScheduleItem.java @@ -20,6 +20,7 @@ import lombok.Data; import java.io.Serializable; @Data +@Deprecated public class CustomTimeScheduleItem implements Serializable { private boolean enabled; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java index fb8488c58e..fcbece4d4b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java @@ -28,6 +28,7 @@ import java.util.TreeMap; @Schema @Data +@Deprecated public class DeviceProfileAlarm implements Serializable { @Schema(description = "String value representing the alarm rule id", example = "highTemperatureAlarmID") diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DurationAlarmConditionSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DurationAlarmConditionSpec.java index e114ec1ddc..361ba1e4b3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DurationAlarmConditionSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DurationAlarmConditionSpec.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; @Data @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class DurationAlarmConditionSpec implements AlarmConditionSpec { private TimeUnit unit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/RepeatingAlarmConditionSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/RepeatingAlarmConditionSpec.java index f9e3fd6d05..75c07dbe02 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/RepeatingAlarmConditionSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/RepeatingAlarmConditionSpec.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue; @Data @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class RepeatingAlarmConditionSpec implements AlarmConditionSpec { private FilterPredicateValue predicate; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SimpleAlarmConditionSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SimpleAlarmConditionSpec.java index 05c8d0df70..4243946a30 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SimpleAlarmConditionSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SimpleAlarmConditionSpec.java @@ -20,6 +20,7 @@ import lombok.Data; @Data @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class SimpleAlarmConditionSpec implements AlarmConditionSpec { @Override public AlarmConditionSpecType getType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java index e46d5edbf3..a8b47db1ab 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SpecificTimeSchedule.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.query.DynamicValue; import java.util.Set; @Data +@Deprecated public class SpecificTimeSchedule implements AlarmSchedule { private String timezone; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java b/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java index f942bc2196..e2949c96eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java @@ -38,7 +38,7 @@ public enum TbMsgType { ENTITY_UNASSIGNED("Entity Unassigned"), ATTRIBUTES_UPDATED("Attributes Updated"), ATTRIBUTES_DELETED("Attributes Deleted"), - ALARM, + ALARM("Alarm"), ALARM_ACK("Alarm Acknowledged"), ALARM_CLEAR("Alarm Cleared"), ALARM_DELETE, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmEvalResult.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmEvalResult.java index 6062b9ea9c..68e3c7e2e7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmEvalResult.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmEvalResult.java @@ -15,6 +15,7 @@ */ package org.thingsboard.rule.engine.profile; +@Deprecated public enum AlarmEvalResult { FALSE, NOT_YET_TRUE, TRUE; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 1707f64bc7..1a5832fccf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -55,6 +55,7 @@ import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithout @Data @Slf4j + class AlarmRuleState { private final AlarmSeverity severity; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java index c6cc39916e..219918c485 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java @@ -48,6 +48,7 @@ import java.util.function.BiFunction; @Data @Slf4j +@Deprecated class AlarmState { public static final String ERROR_MSG = "Failed to process alarm rule for Device [%s]: %s"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java index 33a6fe1631..b9ca93377f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +@Deprecated class DataSnapshot { private volatile boolean ready; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java index 4bd81050db..f78cd7c088 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java @@ -71,6 +71,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE import static org.thingsboard.server.common.data.msg.TbMsgType.TIMESERIES_UPDATED; @Slf4j +@Deprecated class DeviceState { private final boolean persistState; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtx.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtx.java index 3c2884d616..f6cdc9a781 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtx.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtx.java @@ -15,6 +15,7 @@ */ package org.thingsboard.rule.engine.profile; +@Deprecated public interface DynamicPredicateValueCtx { EntityKeyValue getTenantValue(String key); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java index 18fa81f63a..8962dff463 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.ExecutionException; @Slf4j +@Deprecated public class DynamicPredicateValueCtxImpl implements DynamicPredicateValueCtx { private final TenantId tenantId; private CustomerId customerId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/EntityKeyValue.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/EntityKeyValue.java index 7561c51386..9742ffa234 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/EntityKeyValue.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/EntityKeyValue.java @@ -20,6 +20,7 @@ import lombok.Getter; import org.thingsboard.server.common.data.kv.DataType; @EqualsAndHashCode +@Deprecated class EntityKeyValue { @Getter diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java index fffc66d02a..982a7f3b36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java @@ -45,6 +45,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +@Deprecated class ProfileState { private DeviceProfile deviceProfile; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java index d41a42efa3..08af665038 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import java.util.Set; +@Deprecated class SnapshotUpdate { @Getter diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java index 2bb172bd5b..5a146931ff 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java @@ -51,17 +51,18 @@ import java.util.concurrent.TimeUnit; @Slf4j @RuleNode( type = ComponentType.ACTION, - name = "device profile", + name = "device profile (deprecated)", // TODO: add description on why is it deprecated and what to use customRelations = true, relationTypes = {"Alarm Created", "Alarm Updated", "Alarm Severity Updated", "Alarm Cleared", "Success", "Failure"}, version = 1, configClazz = TbDeviceProfileNodeConfiguration.class, - nodeDescription = "Process device messages based on device profile settings", + nodeDescription = "Process device messages based on device profile settings (deprecated)", nodeDetails = "Create and clear alarms based on alarm rules defined in device profile. The output relation type is either " + "'Alarm Created', 'Alarm Updated', 'Alarm Severity Updated' and 'Alarm Cleared' or simply 'Success' if no alarms were affected.", configDirective = "tbActionNodeDeviceProfileConfig", docUrl = "https://thingsboard.io/docs/user-guide/rule-engine-2-0/nodes/action/device-profile/" ) +@Deprecated public class TbDeviceProfileNode implements TbNode { private TbDeviceProfileNodeConfiguration config; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeConfiguration.java index a3180893d1..0605eed516 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeConfiguration.java @@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.NodeConfiguration; @Data @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class TbDeviceProfileNodeConfiguration implements NodeConfiguration { private boolean persistAlarmRulesState; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmRuleState.java index 30aa4c443b..035d564f65 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmRuleState.java @@ -22,6 +22,7 @@ import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor +@Deprecated public class PersistedAlarmRuleState { private long lastEventTs; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmState.java index dba8ba17a8..16d11485df 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedAlarmState.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.alarm.AlarmSeverity; import java.util.Map; @Data +@Deprecated public class PersistedAlarmState { private Map createRuleStates; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedDeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedDeviceState.java index 46f8a3b2ca..d4307e3955 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedDeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/state/PersistedDeviceState.java @@ -20,6 +20,7 @@ import lombok.Data; import java.util.Map; @Data +@Deprecated public class PersistedDeviceState { Map alarmStates; From f667f0f23f796492050e336f1fee37e8495a69d0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Oct 2025 16:58:00 +0300 Subject: [PATCH 040/122] (wip) aggregation cf --- .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 154 +++++++-- ...alculatedFieldManagerMessageProcessor.java | 320 +++++++++++++++++- .../CalculatedFieldRelatedEntityMsg.java | 49 +++ ...tractCalculatedFieldProcessingService.java | 97 ++++++ .../service/cf/CalculatedFieldCache.java | 3 + .../cf/CalculatedFieldProcessingService.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 28 ++ ...faultCalculatedFieldProcessingService.java | 8 + .../DefaultCalculatedFieldQueueService.java | 39 ++- .../service/cf/ctx/state/ArgumentEntry.java | 10 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 140 ++++++++ .../cf/ctx/state/CalculatedFieldState.java | 5 +- .../ctx/state/SingleValueArgumentEntry.java | 8 +- .../state/aggregation/AggArgumentEntry.java | 72 ++++ .../aggregation/AggSingleArgumentEntry.java | 83 +++++ ...ValuesAggregationCalculatedFieldState.java | 161 +++++++++ .../service/cf/ctx/state/aggregation/agg.json | 65 ++++ .../state/aggregation/function/AggEntry.java | 45 +++ .../function/AggFunctionFactory.java | 33 ++ .../aggregation/function/AvgAggEntry.java | 45 +++ .../aggregation/function/BaseAggEntry.java | 52 +++ .../aggregation/function/CountAggEntry.java | 41 +++ .../function/CountUniqueAggEntry.java | 45 +++ .../aggregation/function/MaxAggEntry.java | 40 +++ .../aggregation/function/MinAggEntry.java | 40 +++ .../aggregation/function/SumAggEntry.java | 42 +++ .../entitiy/EntityStateSourcingListener.java | 11 + .../queue/DefaultTbClusterService.java | 40 ++- .../processing/AbstractConsumerService.java | 18 +- .../utils/CalculatedFieldArgumentUtils.java | 2 + .../server/utils/CalculatedFieldUtils.java | 48 ++- ...tValuesAggregationCalculatedFieldTest.java | 314 +++++++++++++++++ .../server/controller/AbstractWebTest.java | 10 + .../server/cluster/TbClusterService.java | 8 + .../server/dao/relation/RelationService.java | 8 + .../common/data/cf/CalculatedFieldType.java | 3 +- .../CalculatedFieldConfiguration.java | 4 +- .../aggregation/AggFunction.java | 20 ++ .../aggregation/AggFunctionInput.java | 34 ++ .../configuration/aggregation/AggInput.java | 36 ++ .../aggregation/AggKeyInput.java | 34 ++ .../configuration/aggregation/AggMetric.java | 31 ++ .../configuration/aggregation/AggSource.java | 30 ++ .../aggregation/CfAggTrigger.java | 104 ++++++ ...gregationCalculatedFieldConfiguration.java | 52 +++ .../server/common/msg/MsgType.java | 2 + .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 + common/proto/src/main/proto/queue.proto | 7 + .../script/api/tbel/TbelCfArg.java | 1 + .../tbel/TbelCfLatestValuesAggregation.java | 44 +++ .../dao/relation/BaseRelationService.java | 34 ++ .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 10 + .../dao/sql/relation/RelationRepository.java | 39 ++- 59 files changed, 2527 insertions(+), 70 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java create mode 100644 application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cababd4b6d..ed4131c114 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -73,6 +73,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DELETE_MSG: processor.process((CalculatedFieldEntityDeleteMsg) msg); break; + case CF_RELATED_ENTITY_MSG: + processor.process((CalculatedFieldRelatedEntityMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ccbdcc6b33..98b5eba4c8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,6 +52,9 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -66,7 +69,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -189,7 +194,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - public void process(CalculatedFieldEntityDeleteMsg msg) { + public void process(CalculatedFieldEntityDeleteMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { if (states.isEmpty()) { @@ -200,16 +205,74 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + EntityId msgEntityId = msg.getEntityId(); + if (msgEntityId instanceof CalculatedFieldId cfId) { + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); + } } else { - msg.getCallback().onSuccess(); + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } + for (Map.Entry entry : states.entrySet()) { + LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); + state.getArguments().forEach((argName, argEntry) -> { + AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; + aggArgEntry.getAggInputs().remove(msgEntityId); + }); + state.getInputs().remove(msgEntityId); + state.setLastMetricsEvalTs(-1); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } } } } + public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); + CalculatedFieldCtx cfCtx = msg.getCalculatedField(); + var state = states.get(cfCtx.getCfId()); + Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + try { + if (state == null) { + state = createState(cfCtx); + } else { + state.setCtx(cfCtx, actorCtx); + } + if (state.isSizeOk()) { + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + state.update(fetchedArguments, cfCtx); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); + states.put(cfCtx.getCfId(), state); + processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + } + } + + @SneakyThrows + private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentsFuture.get(1, TimeUnit.MINUTES); + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -462,59 +525,78 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(ctx.getMainEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(ctx.getLinkedAndDynamicArgs(entityId), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + if (!aggArgNames.isEmpty()) { + for (Map.Entry entry : aggArgNames.entrySet()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); + } + } } - key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + } + if (!argNames.isEmpty()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } } } return arguments; } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); + Map aggregationInputs = ctx.getAggregationInputs(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - for (AttributeValueProto item : attrDataList) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { - continue; + if (!argNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName == null) { + continue; + } + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + continue; + } + arguments.put(argName, new SingleValueArgumentEntry(item)); } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); - continue; + } + if (!aggArgNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + for (Map.Entry entry : aggArgNames.entrySet()) { + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); + } + } } - arguments.put(argName, new SingleValueArgumentEntry(item)); } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4675821a5b..8a717c3cdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -25,18 +25,27 @@ import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMs import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -44,10 +53,13 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -64,6 +76,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -82,6 +96,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private final Map cfTriggers = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -90,6 +105,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final DeviceService deviceService; private final AssetService assetService; private final CustomerService customerService; + private final RelationService relationService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; @@ -107,6 +123,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.deviceService = systemContext.getDeviceService(); this.assetService = systemContext.getAssetService(); this.customerService = systemContext.getCustomerService(); + this.relationService = systemContext.getRelationService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); @@ -129,6 +146,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } + cfTriggers.clear(); ctx.stop(ctx.getSelf()); } @@ -233,6 +251,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityProfileCache.add(profileId, entityId); } updateEntityOwner(entityId); + + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + if (!cfsRelatedToEntity.isEmpty()) { + MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); + cfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + if (!isMyPartition(entityId, callback)) { return; } @@ -240,11 +272,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } @@ -254,33 +286,180 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); + if (fieldsWithRelatedEntityCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); + var entityId = msg.getEntityId(); + oldCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); + }); + newCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); + } else if (msg.isRelationChanged()) { + onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + if (msg.isRelationChanged()) { + onRelationDeleted(msg, callback); + } else { + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + + getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); + }); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); + } + } + + private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); } } @@ -302,6 +481,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -333,6 +515,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -417,6 +602,106 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + // process all aggregation cfs (if any); + List aggregationCalculatedFields = filterAggregationCfs(msg); + if (!aggregationCalculatedFields.isEmpty()) { + cfExecService.pushMsgToLinks(msg, aggregationCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + + List aggregationCalculatedFields = cfTriggers.entrySet().stream() + .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggregationCalculatedFields) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); + }; + if (targetEntity != null) { + filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); + } + } + } + return filteredByRelationCfs; + } + + private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { + List aggCFsUsedProfile = cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matchesProfile(profileId)) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggCFsUsedProfile) { + CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); + if (calculatedFieldEntityCtxId != null) { + filteredByRelationCfs.add(cf); + } + } + return filteredByRelationCfs; + } + + private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + yield entityRelation == null ? null : entityRelation.getFrom(); + } + case TO -> { + EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + yield entityRelation == null ? null : entityRelation.getTo(); + } + }; + if (targetEntity != null) { + return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); + } + } + return null; + } + + private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); + } else { + return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } } public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { @@ -536,6 +821,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + } + + private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { + log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + } + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { log.debug("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); @@ -614,6 +909,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java new file mode 100644 index 0000000000..d3bdd9cc5c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldCtx calculatedField; + private final TbCallback callback; + + public CalculatedFieldRelatedEntityMsg(TenantId tenantId, + EntityId entityId, + CalculatedFieldCtx calculatedField, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedField = calculatedField; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_RELATED_ENTITY_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index d17148a502..304b747f26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,10 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -36,6 +39,8 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; @@ -44,7 +49,9 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +105,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } + case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -114,6 +122,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } + private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + RelationPathLevel relation = aggSource.getRelation(); + return switch (relation.direction()) { + case FROM -> aggSource.getEntityProfiles().stream() + .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) + .flatMap(Collection::stream) + .map(EntityRelation::getTo) + .toList(); + case TO -> + aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); + }; + } + protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -176,6 +197,50 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } + private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); + }); + return futures; + } + + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() + .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + + protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + + ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); + futures.put(key, argumentEntryListenableFuture); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -244,6 +309,38 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } + protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); + case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); + case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); + }; + } + + private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); + var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); + return Futures.transform(attributeOptFuture, attrOpt -> { + log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + + protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + String key = argument.getRefEntityKey().getKey(); + log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); + return Futures.transform( + timeseriesService.findLatest(tenantId, entityId, key), + result -> { + log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index cc77913f4b..e32ca42f9c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -38,6 +39,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); + List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index a9139572b8..4b3e994f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,6 +33,8 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 36210d7302..ae8238fb8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -41,6 +43,8 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -48,6 +52,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Collectors; @Service @Slf4j @@ -68,6 +73,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -81,6 +87,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } } }); calculatedFields.values().forEach(cf -> { @@ -146,6 +155,16 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { + return cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) + .map(Map.Entry::getKey) + .map(this::getCalculatedFieldCtx) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + @Override public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); @@ -155,6 +174,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } + return hasCalculatedFieldsByProfile(tenantId, entityId, filter); + } + + public boolean hasCalculatedFieldsByProfile(TenantId tenantId, EntityId entityId, Predicate filter) { EntityId profileId = getProfileId(tenantId, entityId); if (profileId != null) { List profileCfs = getCalculatedFieldCtxsByEntityId(profileId); @@ -183,6 +206,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); configuration.getReferencedEntities().stream() @@ -214,6 +240,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + cfTriggers.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9b2964a736..b7bd4d87fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -54,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @@ -87,6 +90,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } + @Override + public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + } + @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only scheduledSupported CF instances supports dynamic arguments scheduled updates diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a3e049d1e2..a75df1fb40 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +37,10 @@ import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -71,6 +76,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; + private final RelationService relationService; @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { @@ -81,6 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), + cfTrigger -> cfTrigger.matchesTimeSeries(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -99,6 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), + cfTrigger -> cfTrigger.matchesAttributes(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -116,6 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), + cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -127,6 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), + cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -134,11 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, + Predicate cfAggKeysFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -149,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -176,6 +187,26 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } + List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + for (CalculatedFieldCtx cfCtx : cfCtxs) { + EntityId cfEntityId = cfCtx.getEntityId(); + if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + EntityRelation entityRelation = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + }; + if (entityRelation != null) { + return true; + } + } + } + return false; } @@ -266,6 +297,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } + private boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 2d43883131..f22b10a9c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,6 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -35,7 +37,9 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING") + @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), + @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -66,4 +70,8 @@ public interface ArgumentEntry { return new GeofencingArgumentEntry(entityIdkvEntryMap); } + static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { + return new AggArgumentEntry(entityIdkvEntryMap, false); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 876bfa2a3f..30b92a78d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING + SINGLE_VALUE, TS_ROLLING, GEOFENCING, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e442964280..f75711a107 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -123,7 +123,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, protected void validateNewEntry(String key, ArgumentEntry newEntry) {} - private void updateLastUpdateTimestamp(ArgumentEntry entry) { + protected void updateLastUpdateTimestamp(ArgumentEntry entry) { long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { newTs = singleValueArgumentEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e08abb8b50..963ec3f06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -81,6 +83,8 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; + private final List aggInputs; + private final Map aggregationInputs; private final List argNames; private Output output; private String expression; @@ -122,6 +126,8 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.aggInputs = new ArrayList<>(); + this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -165,6 +171,12 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + aggInputs.addAll(aggConfig.getInputs().values()); + aggregationInputs.putAll(aggConfig.getInputs()); + this.argNames.addAll(aggConfig.getInputs().keySet()); + this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); @@ -199,6 +211,19 @@ public class CalculatedFieldCtx { }); initialized = true; } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + configuration.getMetrics().forEach((key, metric) -> { + if (metric.getInput() instanceof AggFunctionInput functionInput) { + initTbelExpression(functionInput.getFunction()); + } + String filter = metric.getFilter(); + if (filter != null && !filter.isEmpty()) { + initTbelExpression(filter); + } + }); + initialized = true; + } } } @@ -242,6 +267,24 @@ public class CalculatedFieldCtx { return expression.executeScriptAsync(args.toArray()); } + public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(argNames.size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : argNames) { + var arg = entries.get(argName).toTbelCfArg(); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments, latestTimestamp)); + + return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + } + public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg @@ -451,6 +494,25 @@ public class CalculatedFieldCtx { } } + public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return matchesAggKeys(proto.getRemovedTsKeysList()); + } else { + return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() @@ -482,6 +544,67 @@ public class CalculatedFieldCtx { return argNames; } + public boolean matchesAggKeys(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (String key : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggTimeSeries(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (TsKvEntry tsKvEntry : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributes(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (AttributeKvEntry attributeKvEntry : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -499,6 +622,11 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + return true; + } return false; } @@ -517,6 +645,9 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } + if (hasLatestValuesAggregationConfigurationChanges(other)) { + return true; + } return false; } @@ -528,6 +659,15 @@ public class CalculatedFieldCtx { return false; } + private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + } + return false; + } + + public boolean hasRelationQueryDynamicArguments() { return relationQueryDynamicArguments && scheduledUpdateIntervalMillis != -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index cc7188e7a5..bd40ab7a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -32,6 +32,7 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import java.io.Closeable; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -53,6 +54,8 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); + CalculatedFieldCtx getCtx(); + void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); @@ -61,7 +64,7 @@ public interface CalculatedFieldState extends Closeable { void reset(); - ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); + ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception; @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5c1ed32e1d..288b486e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -37,11 +37,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { - private long ts; - private BasicKvEntry kvEntryValue; - private Long version; + protected long ts; + protected BasicKvEntry kvEntryValue; + protected Long version; - private boolean forceResetPrevious; + protected boolean forceResetPrevious; public static final Long DEFAULT_VERSION = -1L; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java new file mode 100644 index 0000000000..138053793f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class AggArgumentEntry implements ArgumentEntry { + + private final Map aggInputs; + + private boolean forceResetPrevious; + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST; + } + + @Override + public Object getValue() { + return aggInputs; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggArgumentEntry aggArgumentEntry) { + aggInputs.putAll(aggArgumentEntry.aggInputs); + return true; + } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { + if (aggSingleArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else { + aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + } + return true; + } else { + throw new IllegalArgumentException("Unsupported argument entry type for aggregation argument entry: " + entry.getType()); + } + } + + @Override + public boolean isEmpty() { + return aggInputs.isEmpty(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java new file mode 100644 index 0000000000..6b81d5380c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AggSingleArgumentEntry extends SingleValueArgumentEntry { + + private EntityId entityId; + private boolean deleted; + + public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + super(ts, kvEntryValue, version); + this.entityId = entityId; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() <= ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + this.entityId = singleValueEntry.getEntityId(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java new file mode 100644 index 0000000000..d25cde020c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +@Slf4j +@Data +public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + + private long lastArgsRefreshTs = -1; + private long lastMetricsEvalTs = -1; + private long deduplicationInterval = -1; + private Map metrics; + + private final Map> inputs = new HashMap<>(); + + public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + metrics = configuration.getMetrics(); + deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + } + + @Override + public void reset() { // must reset everything dependent on arguments + super.reset(); + lastArgsRefreshTs = -1; + lastMetricsEvalTs = -1; + metrics = null; + } + + @Override + public void init() { + super.init(); +// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); +// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = super.update(argumentValues, ctx); + lastArgsRefreshTs = System.currentTimeMillis(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); + Map aggInputs = aggArgumentEntry.getAggInputs(); + aggInputs.forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return updatedArguments; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; + if (intervalPassed && argsUpdatedDuringInterval) { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); + + AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); + + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggMetric.update(arg); + } + } + } + + aggMetric.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); + }); + } + Output output = ctx.getOutput(); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(aggResult) + .build()); + } else { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + } + + private boolean applyAggregation(String filter, Map entityInputs) throws Exception { + if (filter == null || filter.isEmpty()) { + return true; + } else { + Object filterResult = ctx.evaluateTbelExpression(filter, entityInputs, getLatestTimestamp()).get(); + return filterResult instanceof Boolean booleanResult && booleanResult; + } + } + + private Object resolveAggregationInput(AggInput aggInput, Map entityInputs) throws Exception { + if (aggInput instanceof AggFunctionInput functionInput) { + return ctx.evaluateTbelExpression(functionInput.getFunction(), entityInputs, getLatestTimestamp()).get(); + } else { + String inputKey = ((AggKeyInput) aggInput).getKey(); + return entityInputs.get(inputKey).getValue(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json new file mode 100644 index 0000000000..d402f23c6e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -0,0 +1,65 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET", + "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "source": { + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "entityProfiles": [ + { + "entityType": "DEVICE_PROFILE", + "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" + } + ] + }, + "inputs": { + "oc": { + "key": "occupied", + "type": "TS_LATEST" + } + }, + "deduplicationIntervalMillis": 10000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java new file mode 100644 index 0000000000..4239e94fec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AvgAggEntry.class, name = "AVG"), + @JsonSubTypes.Type(value = CountAggEntry.class, name = "COUNT"), + @JsonSubTypes.Type(value = CountUniqueAggEntry.class, name = "COUNT_UNIQUE"), + @JsonSubTypes.Type(value = MaxAggEntry.class, name = "MAX"), + @JsonSubTypes.Type(value = MinAggEntry.class, name = "MIN"), + @JsonSubTypes.Type(value = SumAggEntry.class, name = "SUM") +}) +public interface AggEntry { + + AggFunction getType(); + + void update(Object value); + + Optional result(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java new file mode 100644 index 0000000000..5ccc355b1f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class AggFunctionFactory { + + public static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java new file mode 100644 index 0000000000..ad1f2ee8a8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class AvgAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + private long count = 0L; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + this.count++; + } + + @Override + protected double prepareResult() { + return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.AVG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java new file mode 100644 index 0000000000..0c12bd13c0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import java.util.Optional; + +public abstract class BaseAggEntry implements AggEntry { + + private boolean hasResult = false; + + @Override + public void update(Object value) { + doUpdate(extractDoubleValue(value)); + hasResult = true; + } + + @Override + public Optional result() { + if (hasResult) { + hasResult = false; + return Optional.of(prepareResult()); + } else { + return Optional.empty(); + } + } + + protected abstract void doUpdate(double value); + + protected abstract double prepareResult(); + + protected double extractDoubleValue(Object value) { + try { + return Double.parseDouble(value.toString()); + } catch (Exception e) { + throw new NumberFormatException("Cannot parse value " + value.toString()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java new file mode 100644 index 0000000000..469048d62d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +public class CountAggEntry implements AggEntry { + + private long count = 0L; + + @Override + public void update(Object value) { + count++; + } + + @Override + public Optional result() { + return Optional.of(count); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java new file mode 100644 index 0000000000..b8b0b92470 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class CountUniqueAggEntry implements AggEntry { + + private Set items; + + @Override + public void update(Object value) { + if (value != null) { + items.add(JacksonUtil.toString(value)); + } + } + + @Override + public Optional result() { + return Optional.of(items.size()); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT_UNIQUE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java new file mode 100644 index 0000000000..6e4235b72f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MaxAggEntry extends BaseAggEntry { + + private double max = -Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value > max) { + max = value; + } + } + + @Override + protected double prepareResult() { + return max; + } + + @Override + public AggFunction getType() { + return AggFunction.MAX; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java new file mode 100644 index 0000000000..eeacc3a7d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MinAggEntry extends BaseAggEntry { + + private double min = Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value < min) { + min = value; + } + } + + @Override + protected double prepareResult() { + return min; + } + + @Override + public AggFunction getType() { + return AggFunction.MIN; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java new file mode 100644 index 0000000000..b90817d784 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; + +public class SumAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + } + + @Override + protected double prepareResult() { + return sum.doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.SUM; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index b9fd38f1e2..f762cf5be5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; @@ -271,6 +273,15 @@ public class EntityStateSourcingListener { } } + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + tbClusterService.onRelationUpdated(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + tbClusterService.onRelationDeleted(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } + } + private void onTenantUpdate(Tenant tenant, ComponentLifecycleEvent lifecycleEvent) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), lifecycleEvent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f1494658a8..22faeaf61b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -369,6 +370,17 @@ public class DefaultTbClusterService implements TbClusterService { broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); } + @Override + public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityId) + .profileId(profileId) + .event(state) + .build()); + } + @Override public void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback) { boolean isFirmwareChanged = false; @@ -420,13 +432,13 @@ public class DefaultTbClusterService implements TbClusterService { gatewayNotificationsService.onDeviceDeleted(device); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); - broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, deviceId, device.getDeviceProfileId(), ComponentLifecycleEvent.DELETED); } @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, assetId, asset.getAssetProfileId(), ComponentLifecycleEvent.DELETED); } @Override @@ -723,6 +735,30 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); } + @Override + public void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.UPDATED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + + @Override + public void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.DELETED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 7382ef1c4d..761b69024f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,24 +184,34 @@ public abstract class AbstractConsumerService new ScriptCalculatedFieldState(entityId); case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 4319f72448..d4ee7bb2e3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -45,12 +46,16 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -92,7 +97,10 @@ public class CalculatedFieldUtils { .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { + aggArgumentEntry.getAggInputs() + .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); @@ -130,6 +138,17 @@ public class CalculatedFieldUtils { return ruleState; } + public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { + AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() + .setEntityId(ProtoUtils.toProto(entityId)); + + if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } + + return builder.build(); + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -187,6 +206,7 @@ public class CalculatedFieldUtils { case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -212,11 +232,37 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + Map> arguments = new HashMap<>(); + proto.getAggArgumentsList().forEach(argProto -> { + AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + }); + } } return state; } + public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + if (!proto.hasValue()) { + return new AggSingleArgumentEntry(); + } + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + SingleValueArgumentProto singleValueArgument = proto.getValue(); + TsValueProto tsValueProto = singleValueArgument.getValue(); + return new AggSingleArgumentEntry( + entityId, + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), + singleValueArgument.getVersion() + ); + } + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java new file mode 100644 index 0000000000..1a5fc2e3d3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -0,0 +1,314 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; + +@DaoSqlTest +public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { + + private Tenant savedTenant; + + private DeviceProfile deviceProfile; + private Device device1; + private String accessToken1 = "1234567890111"; + private Device device2; + private String accessToken2 = "1234567890222"; + + private AssetProfile assetProfile; + private Asset asset; + + private CalculatedField calculatedField; + + private long deduplicationInterval = 10000; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant@thingsboard.org"); + tenantAdmin.setFirstName("John"); + tenantAdmin.setLastName("Doe"); + + createUserAndLogin(tenantAdmin, "testPassword"); + + deviceProfile = doPost("/api/deviceProfile", createDeviceProfile("Device Profile"), DeviceProfile.class); + device1 = createDevice("Device 1", deviceProfile.getId(), accessToken1); + device2 = createDevice("Device 2", deviceProfile.getId(), accessToken2); + + postTelemetry(device1.getId(), "{\"occupied\":true}"); + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + assetProfile = doPost("/api/assetProfile", createAssetProfile("Asset Profile"), AssetProfile.class); + asset = createAsset("Asset", assetProfile.getId()); + + createEntityRelation(asset.getId(), device1.getId(), "Contains"); + createEntityRelation(asset.getId(), device2.getId(), "Contains"); + + calculatedField = createOccupancyCF(asset.getId(), List.of(deviceProfile.getId())); + + checkInitialCalculation(); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testChangeProfile_checkMetricsCalculation() throws Exception { + DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); + device1.setDeviceProfileId(deviceProfile2.getId()); + device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + @Test + public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + }); + } + + @Test + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + private void checkInitialCalculation() { + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + } + + private void checkInitialCalculationValues() throws Exception { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + } + + private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + + return createAggCf("Occupied spaces", entityId, + buildSource(EntitySearchDirection.FROM, "Contains", profiles), + Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + aggMetrics, + output); + } + + private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { + AggSource source = new AggSource(); + source.setRelation(new RelationPathLevel(direction, relationType)); + source.setEntityProfiles(profiles); + return source; + } + + private CalculatedField createAggCf(String name, + EntityId entityId, + AggSource aggSource, + Map inputs, + Map metrics, + Output output) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(name); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + + LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + configuration.setSource(aggSource); + configuration.setInputs(inputs); + configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setMetrics(metrics); + configuration.setOutput(output); + + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private Device createDevice(String name, DeviceProfileId deviceProfileId, String accessToken) { + Device device = new Device(); + device.setName(name); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + device.setDeviceData(deviceData); + return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 2050e1075c..6c6b9c44ec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -1063,6 +1063,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { doPost("/api/relation", relation); } + protected void deleteEntityRelation(EntityRelation entityRelation) throws Exception { + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + entityRelation.getFrom().getId(), + entityRelation.getFrom().getEntityType(), + entityRelation.getType(), + entityRelation.getTo().getId(), + entityRelation.getTo().getEntityType()); + doDelete(url); + } + protected List findRelationsByTo(EntityId entityId) throws Exception { String url = String.format("/api/relations?toId=%s&toType=%s", entityId.getId(), entityId.getEntityType().name()); MvcResult mvcResult = doGet(url).andReturn(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 701c22587b..6c8acbfb83 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -30,6 +31,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -87,6 +89,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state); + void broadcast(ComponentLifecycleMsg componentLifecycleMsg); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -137,4 +141,8 @@ public interface TbClusterService extends TbQueueClusterService { void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + + void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..5b3290c110 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,14 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); + + EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + + void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + + void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 7052ab70e9..397d31650a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -24,7 +24,8 @@ public enum CalculatedFieldType { SIMPLE, SCRIPT, GEOFENCING, - ALARM; + ALARM, + LATEST_VALUES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..4a4e3380f5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,7 +41,8 @@ import java.util.stream.Collectors; @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), + @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java new file mode 100644 index 0000000000..cd0e3f66d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +public enum AggFunction { + MIN, MAX, SUM, AVG, COUNT, COUNT_UNIQUE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java new file mode 100644 index 0000000000..f6df80952e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggFunctionInput implements AggInput { + + private String function; + + @Override + public String getType() { + return "function"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java new file mode 100644 index 0000000000..fac988d5d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AggKeyInput.class, name = "key"), + @JsonSubTypes.Type(value = AggFunctionInput.class, name = "function") +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AggInput { + + String getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java new file mode 100644 index 0000000000..1a00a18a9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggKeyInput implements AggInput { + + private String key; + + @Override + public String getType() { + return "key"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java new file mode 100644 index 0000000000..ebd612b1e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AggMetric { + + private AggFunction function; + private String filter; + private AggInput input; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java new file mode 100644 index 0000000000..84f6f92eb3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.RelationPathLevel; + +import java.util.List; + +@Data +public class AggSource { + + private RelationPathLevel relation; + private List entityProfiles; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java new file mode 100644 index 0000000000..393697877e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; +import java.util.function.Predicate; + +@Data +@Builder +public class CfAggTrigger { + + private List entityProfiles; + private List inputs; + + public boolean matches(EntityId profileId, Predicate cfAggTrigger) { + if (matchesProfile(profileId)) { + return cfAggTrigger.test(this); + } + return false; + } + + public boolean matchesProfile(EntityId profileId) { + return entityProfiles.contains(profileId); + } + + public boolean matchesTimeSeries(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + for (TsKvEntry tsKvEntry : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesAttributes(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + for (AttributeKvEntry attributeKvEntry : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesTimeSeriesKeys(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + + for (String key : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + + for (String key : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e3db50fda0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; + +import java.util.List; +import java.util.Map; + +@Data +public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + private AggSource source; + private Map inputs; + private long deduplicationIntervalMillis; + private Map metrics; + private Output output; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public void validate() { + } + + public CfAggTrigger buildTrigger() { + return CfAggTrigger.builder() + .inputs(List.copyOf(inputs.values())) + .entityProfiles(source.getEntityProfiles()) + .build(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index c13b0200c7..f82de35819 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -152,6 +152,8 @@ public enum MsgType { CF_ENTITY_INIT_CF_MSG, CF_ENTITY_DELETE_MSG, + CF_RELATED_ENTITY_MSG, + CF_ARGUMENT_RESET_MSG, // Sent to reset argument; CF_REEVALUATE_MSG; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 23b9fe08e3..38df529c96 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,14 +47,15 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; + private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, null); + this(tenantId, entityId, event, null, null, null, null, false, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -63,6 +64,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; + this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 26a64c7f8a..83fb158efd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,6 +130,7 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); + builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -167,6 +168,7 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); + builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4d99608a6d..764f4c3ec9 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -915,6 +915,11 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } +message AggSingleArgumentEntryProto { + EntityIdProto entityId = 1; + SingleValueArgumentProto value = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -922,6 +927,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; + repeated AggSingleArgumentEntryProto aggArguments = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1292,6 +1298,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; + bool relationChanged = 15; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 73a2183564..bf2123a26b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfTsGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java new file mode 100644 index 0000000000..1b5fa394d2 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfLatestValuesAggregation implements TbelCfArg { + + private final Object value; + + @JsonCreator + public TbelCfLatestValuesAggregation( + @JsonProperty("value") Object value + ) { + this.value = value; + } + + + @Override + public String getType() { + return "LATEST_VALUES_AGGREGATION"; + } + + @Override + public long memorySize() { + return 32; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..df79ca145a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,40 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } + + @Override + public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelation, + relation -> RelationCacheValue.builder().relation(relation).build(), false); + } + + @Override + public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { + RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); + cache.evict(List.of(key)); + log.debug("Processed evict relations by key: {}", key); + } + + @Override + public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + List keys = new ArrayList<>(2); + keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); + keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); + cache.evict(keys); + log.debug("Processed evict relations by keys: {}", keys); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index d6f0525c9d..344af0a6f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,9 +40,14 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; + private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null); + this(from, to, type, typeGroup, null, null); + } + + public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { + this(from, to, type, typeGroup, direction, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..f529396965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -36,8 +36,12 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); + List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..c859b71580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -103,6 +103,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -112,6 +117,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index b4e8a21372..9294236526 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.relation; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -28,6 +27,7 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; +import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -96,4 +96,41 @@ public interface RelationRepository @Param("toId") UUID toId, @Param("toType") String toType, @Param("batchSize") int batchSize); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' + LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' + WHERE r.from_id = :fromId + AND r.from_type = :fromType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + """, nativeQuery = true) + List findByFromAndProfile(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' + LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' + WHERE r.to_id = :toId + AND r.to_type = :toType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + LIMIT 1 + """, nativeQuery = true) + Optional findByToAndProfile(@Param("toId") UUID toId, + @Param("toType") String toType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); } From a585a3b9e69ec1c0cad03c96424db65301caa171 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 13:03:45 +0300 Subject: [PATCH 041/122] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...alculatedFieldManagerMessageProcessor.java | 86 ++++------ ...tractCalculatedFieldProcessingService.java | 158 ++++++++++-------- .../cf/CalculatedFieldProcessingService.java | 2 +- ...faultCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 28 +++- .../state/aggregation/AggArgumentEntry.java | 3 +- .../service/cf/ctx/state/aggregation/agg.json | 4 +- .../processing/AbstractConsumerService.java | 2 - ...tValuesAggregationCalculatedFieldTest.java | 73 +++++++- .../server/dao/relation/RelationService.java | 11 +- .../aggregation/CfAggTrigger.java | 2 +- ...gregationCalculatedFieldConfiguration.java | 2 +- .../ProfileEntityRelationPathQuery.java | 21 +++ .../tbel/TbelCfLatestValuesAggregation.java | 2 +- .../dao/relation/BaseRelationService.java | 98 +++++++++-- .../server/dao/relation/RelationDao.java | 5 +- .../dao/sql/relation/JpaRelationDao.java | 102 ++++++++++- .../dao/sql/relation/RelationRepository.java | 3 +- 19 files changed, 436 insertions(+), 174 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 98b5eba4c8..9dfde4d821 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -69,9 +69,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -264,7 +262,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 8a717c3cdb..3c1e21bcc8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,8 +24,8 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; @@ -255,7 +255,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); if (!cfsRelatedToEntity.isEmpty()) { MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); cfsRelatedToEntity.forEach(ctx -> { @@ -289,8 +289,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); if (fieldsWithRelatedEntityCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); @@ -335,7 +335,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); }); if (isMyPartition(msg.getEntityId(), callback)) { @@ -559,6 +559,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? + cfTriggers.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -613,76 +614,55 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - - List aggregationCalculatedFields = cfTriggers.entrySet().stream() + return cfTriggers.entrySet().stream() .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggregationCalculatedFields) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); - }; - if (targetEntity != null) { - filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); - } - } - } - return filteredByRelationCfs; } - private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { - List aggCFsUsedProfile = cfTriggers.entrySet().stream() + private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { + return cfTriggers.entrySet().stream() .filter(entry -> entry.getValue().matchesProfile(profileId)) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggCFsUsedProfile) { - CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); - if (calculatedFieldEntityCtxId != null) { - filteredByRelationCfs.add(cf); - } - } - return filteredByRelationCfs; } - private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { + List result = new ArrayList<>(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + EntityId cfEntityId = cf.getEntityId(); + EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { + switch (relation.direction()) { case FROM -> { - EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - yield entityRelation == null ? null : entityRelation.getFrom(); + List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByTo != null && !relationsByTo.isEmpty()) { + EntityRelation entityRelation = relationsByTo.get(0); // only one supported + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); + } } case TO -> { - EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); - yield entityRelation == null ? null : entityRelation.getTo(); + List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByFrom != null && !relationsByFrom.isEmpty()) { + for (EntityRelation entityRelation : relationsByFrom) { + if (entityRelation.getTo().equals(cf.getEntityId())) { + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); + } + } + } } - }; - if (targetEntity != null) { - return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); } } - return null; + return result; } private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 304b747f26..db15fc1dc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -51,10 +51,11 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; -import java.util.Collection; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -105,7 +106,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } - case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); + case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -122,17 +123,32 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { RelationPathLevel relation = aggSource.getRelation(); - return switch (relation.direction()) { - case FROM -> aggSource.getEntityProfiles().stream() - .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) - .flatMap(Collection::stream) - .map(EntityRelation::getTo) + + List>> relationListsFut = new ArrayList<>(); + if (aggSource.getEntityProfiles().isEmpty()) { + relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); + } else { + aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); + } + + return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { + if (relationLists == null) { + return new ArrayList<>(); + } + List allRelations = relationLists.stream() + .filter(Objects::nonNull) + .flatMap(List::stream) .toList(); - case TO -> - aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); - }; + + return switch (relation.direction()) { + case FROM -> allRelations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); } protected Map resolveArgumentFutures(Map> argFutures) { @@ -174,6 +190,34 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } + protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); + }); + return futures; + } + + protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); + futures.put(key, argEntryFut); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -197,50 +241,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } - private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); - - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); - }); - return futures; - } - - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { - List>> futures = aggEntities.stream() - .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) - .toList(); - - ListenableFuture>> allFutures = Futures.allAsList(futures); - - return Futures.transform(allFutures, - entries -> ArgumentEntry.createAggArgument( - entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ), - MoreExecutors.directExecutor()); - } - - protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - - ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); - futures.put(key, argumentEntryListenableFuture); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), - MoreExecutors.directExecutor()); - } - private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -265,6 +265,22 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + .map(entityId -> { + ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); + return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + }) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + protected ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument, startTs); @@ -309,7 +325,15 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } - protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { + long maxDataPoints = apiLimitService.getLimit( + tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int argumentLimit = argument.getLimit(); + int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; + return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + } + + private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); @@ -317,36 +341,26 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); - return Map.entry(entityId, entry); + return new AggSingleArgumentEntry(entityId, attributeKvEntry); }, calculatedFieldCallbackExecutor); } - protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { String key = argument.getRefEntityKey().getKey(); log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); return Futures.transform( timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); - return Map.entry(entityId, entry); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); }, calculatedFieldCallbackExecutor); } - private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { - long maxDataPoints = apiLimitService.getLimit( - tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int argumentLimit = argument.getLimit(); - int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; - return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 4b3e994f23..52b3341151 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,7 +33,7 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index b7bd4d87fd..f9ed69a313 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,8 +91,8 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a75df1fb40..44cb2aaee4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,19 +193,27 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); + AggSource source = aggConfig.getSource(); + RelationPathLevel relation = source.getRelation(); EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : calculatedFieldCache.getProfileId(tenantId, cfEntityId); - EntityRelation entityRelation = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + switch (relation.direction()) { + case FROM -> { + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byToAndType.isEmpty()) { + return true; + } + } + case TO -> { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byFromAndType.isEmpty()) { + return true; + } + } }; - if (entityRelation != null) { - return true; - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 138053793f..12ae2c4638 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; @@ -66,7 +67,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return null; + return new TbelCfLatestValuesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json index d402f23c6e..b39092ca74 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET", - "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + "entityType": "ASSET_PROFILE", + "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 761b69024f..0b5fd3a2c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,7 +184,6 @@ public abstract class AbstractConsumerService { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test public void testDeleteRelation_checkMetricsCalculation() throws Exception { deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); @@ -215,6 +260,28 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } +// @Test +// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { +// Device device3 = createDevice("Device 3", "1234567890333"); +// postTelemetry(device3.getId(), "{\"occupied\":true}"); +// createEntityRelation(asset.getId(), device3.getId(), "Contains"); +// +// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); +// configuration.getSource().setEntityProfiles(Collections.emptyList()); +// calculatedField.setConfiguration(configuration); +// saveCalculatedField(calculatedField); +// +// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); +// }); +// } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -229,7 +296,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); @@ -252,7 +319,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); - return createAggCf("Occupied spaces", entityId, + return createAggCf(name, entityId, buildSource(EntitySearchDirection.FROM, "Contains", profiles), Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), aggMetrics, diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 5b3290c110..20348e5922 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -86,11 +87,17 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java index 393697877e..65d545bc6e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -42,7 +42,7 @@ public class CfAggTrigger { } public boolean matchesProfile(EntityId profileId) { - return entityProfiles.contains(profileId); + return entityProfiles.isEmpty() || entityProfiles.contains(profileId); } public boolean matchesTimeSeries(List telemetry) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index e3db50fda0..2760e1855b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -45,7 +45,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public CfAggTrigger buildTrigger() { return CfAggTrigger.builder() .inputs(List.copyOf(inputs.values())) - .entityProfiles(source.getEntityProfiles()) + .entityProfiles(List.copyOf(source.getEntityProfiles())) .build(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java new file mode 100644 index 0000000000..32b338ff6f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.relation; + +import org.thingsboard.server.common.data.id.EntityId; + +public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java index 1b5fa394d2..4d1b42aa94 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -39,6 +39,6 @@ public class TbelCfLatestValuesAggregation implements TbelCfArg { @Override public long memorySize() { - return 32; + return OBJ_SIZE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index df79ca145a..2d584ebebe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,32 +516,92 @@ public class BaseRelationService implements RelationService { } @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelations, - relations -> RelationCacheValue.builder().relations(relations).build(), false); + public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + RelationPathLevel relationPathLevel = relationPathQuery.level(); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + }; } @Override - public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelation, - relation -> RelationCacheValue.builder().relation(relation).build(), false); + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); +// RelationPathLevel relationPathLevel = relationPathQuery.level(); +// return switch (relationPathLevel.direction()) { +// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// }; + } + + @Override + public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); + validate(from); + validateType(relationType); + if (targetProfileId == null) { + return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); + } + + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override - public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { - RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); - cache.evict(List.of(key)); - log.debug("Processed evict relations by key: {}", key); + public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); + validate(to); + validateType(relationType); + if (targetProfileId == null) { + return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); + } + + @Override + public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + +// List keys = new ArrayList<>(5); +// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); +// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); +// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); +// cache.evict(keys); +// log.debug("Processed evict event: {}", event); + List keys = new ArrayList<>(2); keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); @@ -548,6 +609,11 @@ public class BaseRelationService implements RelationService { log.debug("Processed evict relations by keys: {}", keys); } + private void validate(ProfileEntityRelationPathQuery relationPathQuery) { + validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); + relationPathQuery.level().validate(); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index f529396965..6318f1fc9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -40,7 +41,7 @@ public interface RelationDao { List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); List findAllByTo(TenantId tenantId, EntityId to); @@ -78,4 +79,6 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index c859b71580..afecca26c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -41,9 +42,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -118,8 +122,14 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList( + relationRepository.findByToAndProfile( + to.getId(), + to.getEntityType().name(), + typeGroup.name(), + relationType, + profileId.getId())); } @Override @@ -402,4 +412,92 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } + @Override + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { + String sql = buildProfileEntityRelationPathSql(query); + Object[] params = buildProfileEntityRelationPathParams(query); + + log.trace("[{}] profile entity relation path query: {}", tenantId, sql); + + return jdbcTemplate.queryForList(sql, params).stream() + .map(row -> { + var entityRelation = new EntityRelation(); + var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); + var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); + var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); + var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); + var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); + var type = (String) row.get(RELATION_TYPE_PROPERTY); + var version = (Long) row.get(VERSION_COLUMN); + + entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); + entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); + entityRelation.setType(type); + entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); + entityRelation.setVersion(version); + return entityRelation; + }) + .collect(Collectors.toList()); + } + + private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { + final List params = new ArrayList<>(); + + params.add(query.rootEntityId().getId()); + params.add(query.rootEntityId().getEntityType().name()); + + params.add(query.level().relationType()); + + if (query.targetEntityProfileId() != null) { + params.add(query.targetEntityProfileId().getId()); + params.add(query.targetEntityProfileId().getId()); + } + + return params.toArray(); + } + + private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { + EntitySearchDirection direction = query.level().direction(); + + StringBuilder sb = new StringBuilder(); + + sb.append("\n") + .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") + .append(" r.relation_type_group, r.relation_type, r.version\n") + .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); + + sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); + } else { + sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); + } + + sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); + } else { + sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); + } + + if (EntitySearchDirection.FROM == direction) { + sb.append("WHERE r.from_id = ?").append("\n") + .append("AND r.from_type = ?").append("\n"); + } else { + sb.append("WHERE r.to_id = ?").append("\n") + .append("AND r.to_type = ?").append("\n"); + } + + sb.append("AND r.relation_type = ?").append("\n") + .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); + + if (query.targetEntityProfileId() != null) { + sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); + } + + sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); + + return sb.toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 9294236526..0ebd5b6ceb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -126,9 +126,8 @@ public interface RelationRepository AND r.relation_type_group = :relationTypeGroup AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) AND (d.id IS NOT NULL OR a.id IS NOT NULL) - LIMIT 1 """, nativeQuery = true) - Optional findByToAndProfile(@Param("toId") UUID toId, + List findByToAndProfile(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroup") String relationTypeGroup, @Param("relationType") String relationType, From 4731e5bdbea989eb703ad0a4a825ca93874e9f5b Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 14 Oct 2025 16:36:34 +0300 Subject: [PATCH 042/122] UI: Refactoring calculate fields component to support new type --- .../calculated-field.module.ts | 64 +++++ .../calculated-fields-table-config.ts | 8 +- .../calculated-field-dialog.component.html | 195 ++------------- .../calculated-field-dialog.component.ts | 226 +----------------- ...eofencing-zone-groups-panel.component.html | 0 ...eofencing-zone-groups-panel.component.scss | 0 ...-geofencing-zone-groups-panel.component.ts | 0 ...eofencing-zone-groups-table.component.html | 0 ...eofencing-zone-groups-table.component.scss | 0 ...-geofencing-zone-groups-table.component.ts | 8 +- .../geofencing-configuration.component.html | 68 ++++++ .../geofencing-configuration.component.ts | 157 ++++++++++++ .../geofencing-configuration.module.ts | 52 ++++ .../output/caclculate-field-output.module.ts | 36 +++ .../calculated-field-output.component.html | 86 +++++++ .../calculated-field-output.component.ts | 148 ++++++++++++ .../components/public-api.ts | 2 - ...ulated-field-argument-panel.component.html | 0 ...ulated-field-argument-panel.component.scss | 0 ...lculated-field-argument-panel.component.ts | 0 ...lated-field-arguments-table.component.html | 0 ...lated-field-arguments-table.component.scss | 0 ...culated-field-arguments-table.component.ts | 4 +- .../simple-configuration.component.html | 98 ++++++++ .../simple-configuration.component.ts | 205 ++++++++++++++++ .../simple-configuration.module.ts | 48 ++++ .../home/components/home-components.module.ts | 41 ---- .../asset-profile/asset-profile.module.ts | 4 +- .../modules/home/pages/asset/asset.module.ts | 4 +- .../device-profile/device-profile.module.ts | 2 + .../home/pages/device/device.module.ts | 2 + .../components/time-unit-input.component.ts | 2 +- .../shared/models/calculated-field.models.ts | 55 ++++- .../assets/locale/locale.constant-en_US.json | 3 +- 34 files changed, 1063 insertions(+), 455 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => geofencing-configuration}/calculated-field-geofencing-zone-groups-panel.component.html (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => geofencing-configuration}/calculated-field-geofencing-zone-groups-panel.component.scss (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => geofencing-configuration}/calculated-field-geofencing-zone-groups-panel.component.ts (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{geofencing-zone-grups-table => geofencing-configuration}/calculated-field-geofencing-zone-groups-table.component.html (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{geofencing-zone-grups-table => geofencing-configuration}/calculated-field-geofencing-zone-groups-table.component.scss (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{geofencing-zone-grups-table => geofencing-configuration}/calculated-field-geofencing-zone-groups-table.component.ts (97%) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => simple-configuration}/calculated-field-argument-panel.component.html (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => simple-configuration}/calculated-field-argument-panel.component.scss (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{panel => simple-configuration}/calculated-field-argument-panel.component.ts (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{arguments-table => simple-configuration}/calculated-field-arguments-table.component.html (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{arguments-table => simple-configuration}/calculated-field-arguments-table.component.scss (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{arguments-table => simple-configuration}/calculated-field-arguments-table.component.ts (98%) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts new file mode 100644 index 0000000000..9cccf28305 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -0,0 +1,64 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { + CalculatedFieldDialogComponent +} from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { HomeComponentsModule } from '@home/components/home-components.module'; +import { + GeofencingConfigurationModule +} from '@home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module'; +import { + SimpleConfigurationModule +} from '@home/components/calculated-fields/components/simple-configuration/simple-configuration.module'; + +@NgModule({ + declarations: [ + CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, + ], + imports: [ + CommonModule, + SharedModule, + GeofencingConfigurationModule, + EntityDebugSettingsButtonComponent, + HomeComponentsModule, + SimpleConfigurationModule + ], + exports: [ + CalculatedFieldsTableComponent, + ] +}) +export class CalculatedFieldsModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 5f4b894448..1105365347 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -158,9 +158,10 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { @@ -287,6 +288,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + if (calculatedField.type === CalculatedFieldType.GEOFENCING || calculatedField.type === CalculatedFieldType.SIMPLE) { + return of(null); + } const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { const type = calculatedField.configuration.arguments[key].refEntityKey.type; acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index e235b66fa7..222c3ffe91 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -62,187 +62,22 @@ - - @if (fieldFormGroup.get('type').value !== CalculatedFieldType.GEOFENCING) { -
-
{{ 'calculated-fields.arguments' | translate }}
- -
-
-
- {{ (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE ? 'calculated-fields.expression' : 'calculated-fields.type.script' ) | translate }} -
- - -
-
- @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { - - @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { - {{ 'calculated-fields.hint.expression-required' | translate }} - } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { - {{ 'calculated-fields.hint.expression-invalid' | translate }} - } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { - {{ 'calculated-fields.hint.expression-max-length' | translate }} - } - - } @else { - {{ 'calculated-fields.hint.expression' | translate }} - } - -
- -
{{ 'api-usage.tbel' | translate }}
- - -
- -
-
-
- } @else { -
-
- {{ 'calculated-fields.entity-coordinates' | translate }} -
-
- - -
-
- -
-
- {{ 'calculated-fields.geofencing-zone-groups' | translate }} -
- -
- -
- {{ 'calculated-fields.zone-group-refresh-interval' | translate }} -
- -
- - -
-
-
+ @switch (fieldFormGroup.get('type').value) { + @case (CalculatedFieldType.GEOFENCING) { + + } -
-
{{ 'calculated-fields.output' | translate }}
-
- - {{ 'calculated-fields.output-type' | translate }} - - @for (type of outputTypes; track type) { - {{ OutputTypeTranslations.get(type) | translate}} - } - - - @if (outputFormGroup.get('type').value === OutputType.Attribute - && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { - - {{ 'calculated-fields.attribute-scope' | translate }} - - - {{ 'calculated-fields.server-attributes' | translate }} - - - {{ 'calculated-fields.shared-attributes' | translate }} - - - - } -
- @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { -
- - - {{ (outputFormGroup.get('type').value === OutputType.Timeseries - ? 'calculated-fields.timeseries-key' - : 'calculated-fields.attribute-key') - | translate }} - - - @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { - - @if (outputFormGroup.get('name').hasError('required')) { - {{ 'common.hint.key-required' | translate }} - } @else if (outputFormGroup.get('name').hasError('pattern')) { - {{ 'common.hint.key-pattern' | translate }} - } @else if (outputFormGroup.get('name').hasError('maxlength')) { - {{ 'common.hint.key-max-length' | translate }} - } - - } - - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputFormGroup.get('decimalsByDefault').errors && outputFormGroup.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - -
-
- -
- calculated-fields.use-latest-timestamp -
- -
- } -
- + @default { + + + } + }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 8cca16d4ef..cf475111c6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -18,36 +18,25 @@ import { Component, DestroyRef, Inject, ViewEncapsulation } from '@angular/core' import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { FormBuilder, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; import { - ArgumentEntityType, CalculatedField, CalculatedFieldConfiguration, - calculatedFieldDefaultScript, - CalculatedFieldGeofencing, CalculatedFieldTestScriptFn, CalculatedFieldType, - CalculatedFieldTypeTranslations, - getCalculatedFieldArgumentsEditorCompleter, - getCalculatedFieldArgumentsHighlights, - getCalculatedFieldCurrentEntityFilter, - OutputType, - OutputTypeTranslations + CalculatedFieldTypeTranslations } from '@shared/models/calculated-field.models'; -import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; -import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { oneSpaceInsideRegex } from '@shared/models/regex.constants'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith, switchMap } from 'rxjs/operators'; +import { switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { Observable } from 'rxjs'; import { EntityId } from '@shared/models/id/entity-id'; import { AdditionalDebugActionConfig } from '@home/components/entity/debug/entity-debug-settings.model'; -import { EntityFilter } from '@shared/models/query/query.models'; -import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { deepTrim } from '@core/utils'; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -68,70 +57,22 @@ export interface CalculatedFieldDialogData { }) export class CalculatedFieldDialogComponent extends DialogComponent { - readonly minAllowedScheduledUpdateIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedScheduledUpdateIntervalInSecForCF; - fieldFormGroup = this.fb.group({ name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], type: [CalculatedFieldType.SIMPLE], debugSettings: [], - configuration: this.fb.group({ - entityCoordinates: this.fb.group({ - latitudeKeyName: [null, [Validators.required]], - longitudeKeyName: [null, [Validators.required]], - }), - arguments: this.fb.control({}), - zoneGroups: this.fb.control({}), - scheduledUpdateEnabled: [true], - scheduledUpdateInterval: [this.minAllowedScheduledUpdateIntervalInSecForCF], - expressionSIMPLE: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], - expressionSCRIPT: [calculatedFieldDefaultScript], - output: this.fb.group({ - name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], - scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], - type: [OutputType.Timeseries], - decimalsByDefault: [null as number, [Validators.min(0), Validators.max(15), Validators.pattern(digitsRegex)]], - }), - useLatestTs: [false] - }), + configuration: this.fb.control({} as CalculatedFieldConfiguration), }); - functionArgs$ = this.configFormGroup.get('arguments').valueChanges - .pipe( - startWith(this.data.value?.configuration?.arguments ?? {}), - map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) - ); - - argumentsEditorCompleter$ = this.configFormGroup.get('arguments').valueChanges - .pipe( - startWith(this.data.value?.configuration?.arguments ?? {}), - map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj)) - ); - - argumentsHighlightRules$ = this.configFormGroup.get('arguments').valueChanges - .pipe( - startWith(this.data.value?.configuration?.arguments ?? {}), - map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) - ); - additionalDebugActionConfig = this.data.value?.id ? { ...this.data.additionalDebugActionConfig, action: () => this.data.additionalDebugActionConfig.action({ id: this.data.value.id, ...this.fromGroupValue }), } : null; - currentEntityFilter: EntityFilter; - - isRelatedEntity: boolean; - - readonly OutputTypeTranslations = OutputTypeTranslations; - readonly OutputType = OutputType; - readonly AttributeScope = AttributeScope; readonly EntityType = EntityType; readonly CalculatedFieldType = CalculatedFieldType; - readonly ScriptLanguage = ScriptLanguage; readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; - readonly outputTypes = Object.values(OutputType) as OutputType[]; readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; - readonly DataKeyType = DataKeyType; constructor(protected store: Store, protected router: Router, @@ -143,48 +84,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent { const calculatedFieldId = this.data.value?.id?.id; - let testScriptDialogResult$: Observable; - if (calculatedFieldId) { - testScriptDialogResult$ = this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + return this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) .pipe( switchMap(event => { const args = event?.arguments ? JSON.parse(event.arguments) : null; @@ -212,114 +113,13 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + return this.data.getTestScriptDialogFn(this.fromGroupValue, null, false); } private applyDialogData(): void { - const { configuration = {}, type = CalculatedFieldType.SIMPLE, debugSettings = { failuresEnabled: true, allEnabled: true }, ...value } = this.data.value ?? {}; - const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; - const updatedConfig = { ...restConfig , ['expression'+type]: expression }; - this.fieldFormGroup.patchValue({ configuration: updatedConfig, type, debugSettings, ...value }, {emitEvent: false}); - } - - private observeTypeChanges(): void { - this.toggleKeyByCalculatedFieldType(this.fieldFormGroup.get('type').value); - this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); - - this.outputFormGroup.get('type').valueChanges - .pipe(takeUntilDestroyed()) - .subscribe(type => this.toggleScopeByOutputType(type)); - this.fieldFormGroup.get('type').valueChanges - .pipe(takeUntilDestroyed()) - .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); - } - - private observeZoneChanges(): void { - this.configFormGroup.get('zoneGroups').valueChanges - .pipe(takeUntilDestroyed()) - .subscribe((zoneGroups: CalculatedFieldGeofencing) => - this.checkRelatedEntity(zoneGroups) - ); - this.checkRelatedEntity(this.configFormGroup.get('zoneGroups').value); - } - - private observeScheduledUpdateEnabled(): void { - this.configFormGroup.get('scheduledUpdateEnabled').valueChanges - .pipe(takeUntilDestroyed()) - .subscribe((value: boolean) => - this.checkScheduledUpdateEnabled(value) - ); - this.checkScheduledUpdateEnabled(this.configFormGroup.get('scheduledUpdateEnabled').value); - } - - private checkScheduledUpdateEnabled(value: boolean) { - if (value) { - this.configFormGroup.get('scheduledUpdateInterval').enable({emitEvent: false}); - } else { - this.configFormGroup.get('scheduledUpdateInterval').disable({emitEvent: false}); - } - } - - private checkRelatedEntity(zoneGroups: CalculatedFieldGeofencing) { - this.isRelatedEntity = Object.values(zoneGroups).some(zone => zone.refDynamicSourceConfiguration?.type === ArgumentEntityType.RelationQuery); - } - - private toggleScopeByOutputType(type: OutputType): void { - if (type === OutputType.Attribute) { - this.outputFormGroup.get('scope').enable({emitEvent: false}); - } else { - this.outputFormGroup.get('scope').disable({emitEvent: false}); - } - if (this.fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { - if (type === OutputType.Attribute) { - this.configFormGroup.get('useLatestTs').disable({emitEvent: false}); - } else { - this.configFormGroup.get('useLatestTs').enable({emitEvent: false}); - } - } else { - this.configFormGroup.get('useLatestTs').disable({emitEvent: false}); - } - } - - private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { - if (type === CalculatedFieldType.GEOFENCING) { - this.configFormGroup.get('entityCoordinates').enable({emitEvent: false}); - this.configFormGroup.get('zoneGroups').enable({emitEvent: false}); - this.configFormGroup.get('scheduledUpdateInterval').enable({emitEvent: false}); - - this.outputFormGroup.get('name').disable({emitEvent: false}); - this.configFormGroup.get('useLatestTs').disable({emitEvent: false}); - this.configFormGroup.get('expressionSIMPLE').disable({emitEvent: false}); - this.configFormGroup.get('expressionSCRIPT').disable({emitEvent: false}); - this.configFormGroup.get('arguments').disable({emitEvent: false}); - } else { - this.configFormGroup.get('entityCoordinates').disable({emitEvent: false}); - this.configFormGroup.get('zoneGroups').disable({emitEvent: false}); - this.configFormGroup.get('scheduledUpdateInterval').disable({emitEvent: false}); - - if (type === CalculatedFieldType.SIMPLE) { - this.outputFormGroup.get('name').enable({emitEvent: false}); - this.configFormGroup.get('expressionSIMPLE').enable({emitEvent: false}); - this.configFormGroup.get('expressionSCRIPT').disable({emitEvent: false}); - if (this.outputFormGroup.get('type').value === OutputType.Attribute) { - this.configFormGroup.get('useLatestTs').disable({emitEvent: false}); - } else { - this.configFormGroup.get('useLatestTs').enable({emitEvent: false}); - } - } else { - this.outputFormGroup.get('name').disable({emitEvent: false}); - this.configFormGroup.get('useLatestTs').disable({emitEvent: false}); - this.configFormGroup.get('expressionSIMPLE').disable({emitEvent: false}); - this.configFormGroup.get('expressionSCRIPT').enable({emitEvent: false}); - } - } + const { configuration = {} as CalculatedFieldConfiguration, type = CalculatedFieldType.SIMPLE, debugSettings = { failuresEnabled: true, allEnabled: true }, ...value } = this.data.value ?? {}; + this.fieldFormGroup.patchValue({ configuration, type, debugSettings, ...value }, {emitEvent: false}); } private observeIsLoading(): void { @@ -328,8 +128,6 @@ export class CalculatedFieldDialogComponent extends DialogComponent +
+
+
+ {{ 'calculated-fields.entity-coordinates' | translate }} +
+
+ + +
+
+ +
+
+ {{ 'calculated-fields.geofencing-zone-groups' | translate }} +
+ +
+ +
+ {{ 'calculated-fields.zone-group-refresh-interval' | translate }} +
+ +
+ + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts new file mode 100644 index 0000000000..67c0fe8749 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts @@ -0,0 +1,157 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { + ArgumentEntityType, + CalculatedFieldGeofencing, + CalculatedFieldGeofencingConfiguration, + CalculatedFieldOutput, + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter, + OutputType +} from '@shared/models/calculated-field.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-geofencing-configuration', + templateUrl: './geofencing-configuration.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => GeofencingConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => GeofencingConfigurationComponent), + multi: true + } + ], +}) +export class GeofencingConfigurationComponent implements ControlValueAccessor, Validator, OnInit { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + readonly minAllowedScheduledUpdateIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedScheduledUpdateIntervalInSecForCF; + readonly DataKeyType = DataKeyType; + + geofencingConfiguration = this.fb.group({ + entityCoordinates: this.fb.group({ + latitudeKeyName: [null, [Validators.required]], + longitudeKeyName: [null, [Validators.required]], + }), + zoneGroups: this.fb.control>({}), + scheduledUpdateEnabled: [true], + scheduledUpdateInterval: [this.minAllowedScheduledUpdateIntervalInSecForCF], + output: this.fb.control({scope: AttributeScope.SERVER_SCOPE, type: OutputType.Timeseries}) + }); + + currentEntityFilter: EntityFilter; + isRelatedEntity: boolean; + + private propagateChange: (config: CalculatedFieldGeofencingConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder, + private store: Store) { + + this.geofencingConfiguration.get('zoneGroups').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe((zoneGroups: Record) => + this.checkRelatedEntity(zoneGroups) + ); + + this.geofencingConfiguration.get('scheduledUpdateEnabled').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe((value: boolean) => + this.checkScheduledUpdateEnabled(value) + ); + + this.geofencingConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(() => { + this.updatedModel(this.geofencingConfiguration.getRawValue() as any); + }) + } + + ngOnInit() { + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); + } + + validate(): ValidationErrors | null { + return this.geofencingConfiguration.valid ? null : { geofencingConfigError: false }; + } + + writeValue(config: CalculatedFieldGeofencingConfiguration): void { + this.geofencingConfiguration.patchValue(config, {emitEvent: false}); + this.checkRelatedEntity(this.geofencingConfiguration.get('zoneGroups').value); + this.checkScheduledUpdateEnabled(this.geofencingConfiguration.get('scheduledUpdateEnabled').value); + } + + registerOnChange(fn: (config: CalculatedFieldGeofencingConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.geofencingConfiguration.disable({emitEvent: false}); + } else { + this.geofencingConfiguration.enable({emitEvent: false}); + this.checkScheduledUpdateEnabled(this.geofencingConfiguration.get('scheduledUpdateEnabled').value); + } + } + + private updatedModel(value: CalculatedFieldGeofencingConfiguration) { + value.type = CalculatedFieldType.GEOFENCING; + this.propagateChange(value) + } + + private checkScheduledUpdateEnabled(value: boolean) { + if (value) { + this.geofencingConfiguration.get('scheduledUpdateInterval').enable({emitEvent: false}); + } else { + this.geofencingConfiguration.get('scheduledUpdateInterval').disable({emitEvent: false}); + } + } + + private checkRelatedEntity(zoneGroups: Record) { + this.isRelatedEntity = Object.values(zoneGroups).some(zone => zone.refDynamicSourceConfiguration?.type === ArgumentEntityType.RelationQuery); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts new file mode 100644 index 0000000000..8fc52d2940 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts @@ -0,0 +1,52 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { + CalculatedFieldGeofencingZoneGroupsTableComponent +} from '@home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-table.component'; +import { + CalculatedFieldGeofencingZoneGroupsPanelComponent +} from '@home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component'; +import { SharedModule } from '@shared/shared.module'; +import { + GeofencingConfigurationComponent +} from '@home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/caclculate-field-output.module'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule + ], + declarations: [ + CalculatedFieldGeofencingZoneGroupsTableComponent, + CalculatedFieldGeofencingZoneGroupsPanelComponent, + GeofencingConfigurationComponent + ], + exports: [ + CalculatedFieldGeofencingZoneGroupsTableComponent, + CalculatedFieldGeofencingZoneGroupsPanelComponent, + GeofencingConfigurationComponent + ] +}) +export class GeofencingConfigurationModule { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts new file mode 100644 index 0000000000..83b3970d9b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts @@ -0,0 +1,36 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputComponent +} from '@home/components/calculated-fields/components/output/calculated-field-output.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + ], + declarations: [ + CalculatedFieldOutputComponent + ], + exports: [ + CalculatedFieldOutputComponent + ] +}) +export class CalculatedFieldOutputModule { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html new file mode 100644 index 0000000000..ededb1d78b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html @@ -0,0 +1,86 @@ + +
+
{{ 'calculated-fields.output' | translate }}
+
+ + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate }} + } + + + @if (outputForm.get('type').value === OutputType.Attribute + && (entityId.entityType === EntityType.DEVICE || entityId.entityType === EntityType.DEVICE_PROFILE)) { + + {{ 'calculated-fields.attribute-scope' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + + + } +
+ @if (simpleMode) { +
+ + + {{ + (outputForm.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate + }} + + + @if (outputForm.get('name').errors && outputForm.get('name').touched) { + + @if (outputForm.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputForm.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputForm.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
+ + + + + + + + + + } +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts new file mode 100644 index 0000000000..9a95c921fa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts @@ -0,0 +1,148 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, DestroyRef, forwardRef, inject, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { + CalculatedFieldOutput, + CalculatedFieldSimpleOutput, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-calculate-field-output', + templateUrl: './calculated-field-output.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldOutputComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldOutputComponent), + multi: true + } + ], +}) +export class CalculatedFieldOutputComponent implements ControlValueAccessor, Validator, OnInit, OnChanges { + + @Input() + simpleMode = false; + + @Input({required: true}) + entityId: EntityId; + + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly EntityType = EntityType; + + private fb = inject(FormBuilder); + private destroyRef = inject(DestroyRef); + + outputForm = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + scope: [{value: AttributeScope.SERVER_SCOPE, disabled: true}], + type: [OutputType.Timeseries], + decimalsByDefault: [null as number, [Validators.min(0), Validators.max(15), Validators.pattern(digitsRegex)]], + }); + + private propagateChange: (config: CalculatedFieldOutput | CalculatedFieldSimpleOutput) => void = () => { }; + + ngOnInit() { + this.outputForm.get('type').valueChanges + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(type => this.toggleScopeByOutputType(type)); + + this.updatedFormWithMode(); + + this.outputForm.valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe((value: CalculatedFieldOutput | CalculatedFieldSimpleOutput) => { + this.updatedModel(value) + }) + } + + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const change = changes[propName]; + if (change.currentValue !== change.previousValue) { + if (propName === 'simpleMode') { + this.updatedFormWithMode(); + if (!change.firstChange) { + this.outputForm.updateValueAndValidity(); + } + } + } + } + } + + validate(): ValidationErrors | null { + return this.outputForm.valid ? null : {outputConfig: false}; + } + + writeValue(value: CalculatedFieldOutput | CalculatedFieldSimpleOutput): void { + this.outputForm.patchValue(value, {emitEvent: false}); + this.outputForm.get('type').updateValueAndValidity({onlySelf: true}); + } + + registerOnChange(fn: (config: CalculatedFieldOutput | CalculatedFieldSimpleOutput) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + private updatedModel(value: CalculatedFieldOutput | CalculatedFieldSimpleOutput) { + if (this.simpleMode && 'name' in value) { + value.name = value.name?.trim() ?? ''; + } + this.propagateChange(value); + } + + private toggleScopeByOutputType(type: OutputType): void { + if (type === OutputType.Attribute) { + this.outputForm.get('scope').enable({emitEvent: false}); + } else { + this.outputForm.get('scope').disable({emitEvent: false}); + } + } + + private updatedFormWithMode(): void { + if (this.simpleMode) { + this.outputForm.get('name').enable({emitEvent: false}); + this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); + } else { + this.outputForm.get('name').disable({emitEvent: false}); + this.outputForm.get('decimalsByDefault').disable({emitEvent: false}); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index 9e3c52bc4f..d4d4f9d1da 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -15,7 +15,5 @@ /// export * from './dialog/calculated-field-dialog.component'; -export * from './arguments-table/calculated-field-arguments-table.component'; -export * from './panel/calculated-field-argument-panel.component'; export * from './debug-dialog/calculated-field-debug-dialog.component'; export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.html similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.html diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.scss similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.ts similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.html similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.html diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.scss similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts similarity index 98% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts index 28a3f09126..03730f3c69 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts @@ -43,7 +43,9 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType, } from '@shared/models/calculated-field.models'; -import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html new file mode 100644 index 0000000000..a4c37bcdee --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html @@ -0,0 +1,98 @@ + +
+
+
{{ 'calculated-fields.arguments' | translate }}
+ +
+
+
+ {{ (isScript ? 'calculated-fields.type.script' : 'calculated-fields.expression') | translate }} +
+ + +
+
+ @if (simpleConfiguration.get('expressionSIMPLE').errors && simpleConfiguration.get('expressionSIMPLE').touched) { + + @if (simpleConfiguration.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (simpleConfiguration.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (simpleConfiguration.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } @else { + {{ 'calculated-fields.hint.expression' | translate }} + } + +
+ +
{{ 'api-usage.tbel' | translate }} +
+ + +
+ +
+
+
+ +
+ +
+ calculated-fields.use-latest-timestamp +
+ +
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts new file mode 100644 index 0000000000..42137cac1c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts @@ -0,0 +1,205 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnChanges, SimpleChanges } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { oneSpaceInsideRegex } from '@shared/models/regex.constants'; +import { + calculatedFieldDefaultScript, + CalculatedFieldScriptConfiguration, + CalculatedFieldSimpleConfiguration, + CalculatedFieldSimpleOutput, + CalculatedFieldType, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType +} from '@shared/models/calculated-field.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { deepClone } from '@core/utils'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable } from 'rxjs'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { map } from 'rxjs/operators'; + +type SimpeConfiguration = CalculatedFieldSimpleConfiguration | CalculatedFieldScriptConfiguration; + +@Component({ + selector: 'tb-simple-configuration', + templateUrl: './simple-configuration.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SimpleConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SimpleConfigurationComponent), + multi: true + } + ], +}) +export class SimpleConfigurationComponent implements ControlValueAccessor, Validator, OnChanges { + + @Input() + isScript: boolean; + + @Input() + entityId: EntityId; + + @Input() + tenantId: string; + + @Input() + entityName: string; + + @Input() + testScript$: Observable; + + simpleConfiguration = this.fb.group({ + arguments: this.fb.control({}), + expressionSIMPLE: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + expressionSCRIPT: [calculatedFieldDefaultScript], + output: this.fb.control({ + name: '', + scope: AttributeScope.SERVER_SCOPE, + type: OutputType.Timeseries, + decimalsByDefault: null + }), + useLatestTs: [false] + }); + + readonly ScriptLanguage = ScriptLanguage; + readonly CalculatedFieldType = CalculatedFieldType; + readonly OutputType = OutputType; + + functionArgs$ = this.simpleConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.simpleConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj ?? {})) + ); + + argumentsHighlightRules$ = this.simpleConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + private propagateChange: (config: SimpeConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder) { + this.simpleConfiguration.get('output').valueChanges.pipe( + takeUntilDestroyed(), + ).subscribe(() => { + this.toggleScopeByOutputType(); + }); + + this.simpleConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value) => { + const { expressionSIMPLE, expressionSCRIPT, ...config } = value; + const cfConfig = config as SimpeConfiguration; + cfConfig.expression = this.isScript ? expressionSCRIPT : expressionSIMPLE; + this.updatedModel(cfConfig); + }) + } + + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const change = changes[propName]; + if (change.currentValue !== change.previousValue) { + if (propName === 'isScript') { + this.updatedFormWithScript(); + if (!change.firstChange) { + this.simpleConfiguration.updateValueAndValidity(); + } + } + } + } + } + + validate(): ValidationErrors | null { + return this.simpleConfiguration.valid ? null : {invalidSimpleConfig: false}; + } + + writeValue(value: SimpeConfiguration): void { + const formValue: any = deepClone(value); + if (this.isScript) { + formValue.expressionSCRIPT = formValue.expression; + } else { + formValue.expressionSIMPLE = formValue.expression; + } + this.simpleConfiguration.patchValue(formValue, {emitEvent: false}); + this.simpleConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + this.updatedFormWithScript(); + } + + registerOnChange(fn: (config: SimpeConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { + } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.simpleConfiguration.disable({emitEvent: false}); + } else { + this.simpleConfiguration.enable({emitEvent: false}); + this.updatedFormWithScript(); + } + } + + onTestScript() { + this.testScript$?.subscribe((expression) => { + this.simpleConfiguration.get('expressionSCRIPT').setValue(expression); + this.simpleConfiguration.get('expressionSCRIPT').markAsDirty(); + }) + } + + private updatedModel(value: SimpeConfiguration): void { + value.type = this.isScript ? CalculatedFieldType.SCRIPT : CalculatedFieldType.SIMPLE; + this.propagateChange(value); + } + + private updatedFormWithScript() { + if (this.isScript) { + this.simpleConfiguration.get('expressionSIMPLE').disable({emitEvent: false}); + this.simpleConfiguration.get('expressionSCRIPT').enable({emitEvent: false}); + } else { + this.simpleConfiguration.get('expressionSIMPLE').enable({emitEvent: false}); + this.simpleConfiguration.get('expressionSCRIPT').disable({emitEvent: false}); + } + this.toggleScopeByOutputType(); + } + + private toggleScopeByOutputType(): void { + if (this.isScript || this.simpleConfiguration.get('output').value.type === OutputType.Attribute) { + this.simpleConfiguration.get('useLatestTs').disable({emitEvent: false}); + } else { + this.simpleConfiguration.get('useLatestTs').enable({emitEvent: false}); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts new file mode 100644 index 0000000000..aee32a0916 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts @@ -0,0 +1,48 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + SimpleConfigurationComponent +} from '@home/components/calculated-fields/components/simple-configuration/simple-configuration.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/caclculate-field-output.module'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + ], + declarations: [ + SimpleConfigurationComponent, + CalculatedFieldArgumentPanelComponent, + CalculatedFieldArgumentsTableComponent + ], + exports: [ + SimpleConfigurationComponent + ] +}) +export class SimpleConfigurationModule {} diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index d3816ddaca..2c9c08aeb3 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,36 +183,13 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; -import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; -import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; import { EntityDebugSettingsButtonComponent } from '@home/components/entity/debug/entity-debug-settings-button.component'; -import { - CalculatedFieldArgumentsTableComponent -} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; -import { - CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; -import { - CalculatedFieldDebugDialogComponent -} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; -import { - CalculatedFieldScriptTestDialogComponent -} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; -import { - CalculatedFieldTestArgumentsComponent -} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; import { CheckConnectivityDialogComponent } from '@home/components/ai-model/check-connectivity-dialog.component'; import { AIModelDialogComponent } from '@home/components/ai-model/ai-model-dialog.component'; import { ResourcesDialogComponent } from "@home/components/resources/resources-dialog.component"; import { ResourcesLibraryComponent } from "@home/components/resources/resources-library.component"; -import { - CalculatedFieldGeofencingZoneGroupsTableComponent -} from '@home/components/calculated-fields/components/geofencing-zone-grups-table/calculated-field-geofencing-zone-groups-table.component'; -import { - CalculatedFieldGeofencingZoneGroupsPanelComponent -} from '@home/components/calculated-fields/components/panel/calculated-field-geofencing-zone-groups-panel.component'; @NgModule({ declarations: @@ -357,15 +334,6 @@ import { SendNotificationButtonComponent, EntityChipsComponent, DashboardViewComponent, - CalculatedFieldsTableComponent, - CalculatedFieldDialogComponent, - CalculatedFieldArgumentsTableComponent, - CalculatedFieldArgumentPanelComponent, - CalculatedFieldDebugDialogComponent, - CalculatedFieldScriptTestDialogComponent, - CalculatedFieldTestArgumentsComponent, - CalculatedFieldGeofencingZoneGroupsTableComponent, - CalculatedFieldGeofencingZoneGroupsPanelComponent, CheckConnectivityDialogComponent, AIModelDialogComponent, ResourcesDialogComponent, @@ -508,15 +476,6 @@ import { SendNotificationButtonComponent, EntityChipsComponent, DashboardViewComponent, - CalculatedFieldsTableComponent, - CalculatedFieldDialogComponent, - CalculatedFieldArgumentsTableComponent, - CalculatedFieldArgumentPanelComponent, - CalculatedFieldDebugDialogComponent, - CalculatedFieldScriptTestDialogComponent, - CalculatedFieldTestArgumentsComponent, - CalculatedFieldGeofencingZoneGroupsTableComponent, - CalculatedFieldGeofencingZoneGroupsPanelComponent, CheckConnectivityDialogComponent, AIModelDialogComponent, ResourcesDialogComponent, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts index b85c83f281..c174a2b97b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts @@ -20,6 +20,7 @@ import { SharedModule } from '@shared/shared.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { AssetProfileTabsComponent } from './asset-profile-tabs.component'; import { AssetProfileRoutingModule } from './asset-profile-routing.module'; +import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -29,7 +30,8 @@ import { AssetProfileRoutingModule } from './asset-profile-routing.module'; CommonModule, SharedModule, HomeComponentsModule, - AssetProfileRoutingModule + CalculatedFieldsModule, + AssetProfileRoutingModule, ] }) export class AssetProfileModule { } diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts b/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts index 475af2fb9a..44fa22520f 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts @@ -23,6 +23,7 @@ import { AssetTableHeaderComponent } from './asset-table-header.component'; import { AssetRoutingModule } from './asset-routing.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { AssetTabsComponent } from '@home/pages/asset/asset-tabs.component'; +import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -35,7 +36,8 @@ import { AssetTabsComponent } from '@home/pages/asset/asset-tabs.component'; SharedModule, HomeComponentsModule, HomeDialogsModule, - AssetRoutingModule + CalculatedFieldsModule, + AssetRoutingModule, ] }) export class AssetModule { } diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts index 76d15d00f1..12b68f4ab4 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts @@ -20,6 +20,7 @@ import { SharedModule } from '@shared/shared.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { DeviceProfileTabsComponent } from './device-profile-tabs.component'; import { DeviceProfileRoutingModule } from './device-profile-routing.module'; +import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -29,6 +30,7 @@ import { DeviceProfileRoutingModule } from './device-profile-routing.module'; CommonModule, SharedModule, HomeComponentsModule, + CalculatedFieldsModule, DeviceProfileRoutingModule ] }) diff --git a/ui-ngx/src/app/modules/home/pages/device/device.module.ts b/ui-ngx/src/app/modules/home/pages/device/device.module.ts index 4c74da0f89..8681ff7fe7 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device.module.ts @@ -36,6 +36,7 @@ import { SnmpDeviceTransportConfigurationComponent } from './data/snmp-device-tr import { DeviceCredentialsModule } from '@home/components/device/device-credentials.module'; import { DeviceProfileCommonModule } from '@home/components/profile/device/common/device-profile-common.module'; import { DeviceCheckConnectivityDialogComponent } from './device-check-connectivity-dialog.component'; +import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -61,6 +62,7 @@ import { DeviceCheckConnectivityDialogComponent } from './device-check-connectiv HomeDialogsModule, DeviceCredentialsModule, DeviceProfileCommonModule, + CalculatedFieldsModule, DeviceRoutingModule ] }) diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.ts b/ui-ngx/src/app/shared/components/time-unit-input.component.ts index 44f1be514a..35b64514c7 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.ts +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.ts @@ -178,7 +178,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, this.timeInputForm.disable({emitEvent: false}); } else { this.timeInputForm.enable({emitEvent: false}); - if(this.timeInputForm.invalid) { + if(!this.timeInputForm.valid) { setTimeout(() => this.updatedModel(this.timeInputForm.value, true)) } } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 841baea168..8a2c34d036 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -31,16 +31,35 @@ import { } from '@shared/models/ace/ace.models'; import { EntitySearchDirection } from '@shared/models/relation.models'; -export interface CalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { - configuration: CalculatedFieldConfiguration; - type: CalculatedFieldType; +interface BaseCalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { entityId: EntityId; } +export interface CalculatedFieldSimple extends BaseCalculatedField { + type: CalculatedFieldType.SIMPLE; + configuration: CalculatedFieldSimpleConfiguration; +} + +export interface CalculatedFieldScript extends BaseCalculatedField { + type: CalculatedFieldType.SCRIPT; + configuration: CalculatedFieldScriptConfiguration; +} + +export interface CalculatedFieldGeofencing extends BaseCalculatedField { + type: CalculatedFieldType.GEOFENCING; + configuration: CalculatedFieldGeofencingConfiguration; +} + +export type CalculatedField = + | CalculatedFieldSimple + | CalculatedFieldScript + | CalculatedFieldGeofencing; + export enum CalculatedFieldType { SIMPLE = 'SIMPLE', SCRIPT = 'SCRIPT', - GEOFENCING = 'GEOFENCING' + GEOFENCING = 'GEOFENCING', + PROPAGATION = 'PROPAGATION' } export const CalculatedFieldTypeTranslations = new Map( @@ -48,22 +67,44 @@ export const CalculatedFieldTypeTranslations = new Map; + output: CalculatedFieldSimpleOutput; +} + +export interface CalculatedFieldScriptConfiguration { + type: CalculatedFieldType.SCRIPT; + expression?: string; + arguments?: Record; + output: CalculatedFieldOutput; +} + +export interface CalculatedFieldGeofencingConfiguration { + type: CalculatedFieldType.GEOFENCING; zoneGroups?: Record; + scheduledUpdateEnabled?: boolean; scheduledUpdateInterval?: number; output: CalculatedFieldOutput; } export interface CalculatedFieldOutput { type: OutputType; - name: string; scope?: AttributeScope; +} + +export interface CalculatedFieldSimpleOutput extends CalculatedFieldOutput { + name: string; decimalsByDefault?: number; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 8bba7205e3..24266ad192 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1053,7 +1053,8 @@ "type": { "simple": "Simple", "script": "Script", - "geofencing" : "Geofencing" + "geofencing" : "Geofencing", + "propagation": "Propagation" }, "arguments": "Arguments", "decimals-by-default": "Decimals by default", From 9ac6219882550ee6b138f6d10807096d313e7eda Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 17:05:47 +0300 Subject: [PATCH 043/122] removed entityProfiles from config --- ...CalculatedFieldEntityMessageProcessor.java | 33 ++-- ...alculatedFieldManagerMessageProcessor.java | 143 ++++----------- ...tractCalculatedFieldProcessingService.java | 65 +++---- .../service/cf/CalculatedFieldCache.java | 3 +- .../cf/DefaultCalculatedFieldCache.java | 26 ++- .../DefaultCalculatedFieldQueueService.java | 31 +--- .../service/cf/ctx/state/ArgumentEntry.java | 4 + .../cf/ctx/state/CalculatedFieldCtx.java | 112 ++++-------- .../state/aggregation/function/new_agg.json | 60 +++++++ .../utils/CalculatedFieldArgumentUtils.java | 9 + ...tValuesAggregationCalculatedFieldTest.java | 167 ++++++++++-------- .../aggregation/CfAggTrigger.java | 104 ----------- ...gregationCalculatedFieldConfiguration.java | 19 +- 13 files changed, 297 insertions(+), 479 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9dfde4d821..e253633727 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,8 +171,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } - } + } } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -523,22 +522,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); if (!aggArgNames.isEmpty()) { - for (Map.Entry entry : aggArgNames.entrySet()) { - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); - } + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(originator, item)); } } } @@ -560,17 +558,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getAggregationInputs(); + Map aggregationInputs = ctx.getRelatedEntityArguments(); return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -589,10 +587,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (!aggArgNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - for (Map.Entry entry : aggArgNames.entrySet()) { - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); - } + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3c1e21bcc8..620eee41ee 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -32,20 +31,17 @@ import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -53,13 +49,11 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -76,8 +70,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -96,7 +88,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); - private final Map cfTriggers = new HashMap<>(); + private final Map aggCalculatedFields = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -146,7 +138,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } - cfTriggers.clear(); + aggCalculatedFields.clear(); ctx.stop(ctx.getSelf()); } @@ -252,19 +244,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } updateEntityOwner(entityId); - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); - if (!cfsRelatedToEntity.isEmpty()) { - MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); - cfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - if (!isMyPartition(entityId, callback)) { return; } @@ -272,11 +251,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } @@ -286,35 +265,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); - var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); - if (fieldsWithRelatedEntityCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); - var entityId = msg.getEntityId(); - oldCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); - }); - newCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); @@ -334,10 +294,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - - getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { - applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); - }); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); @@ -365,9 +321,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -386,9 +341,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -424,9 +378,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -445,9 +398,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -482,7 +434,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } calculatedFields.put(cf.getId(), cfCtx); if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -515,8 +467,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); @@ -559,7 +511,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? - cfTriggers.remove(cfId); + aggCalculatedFields.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -614,21 +566,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - return cfTriggers.entrySet().stream() - .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + return aggCalculatedFields.values().stream() + .filter(cf -> cf.relatedEntityMatches(msg.getProto())) .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matchesProfile(profileId)) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + private List getCfsWithRelationToEntity(EntityId entityId) { + return aggCalculatedFields.values().stream() .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); } @@ -636,24 +581,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityId = cf.getEntityId(); - EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = configuration.getRelation(); switch (relation.direction()) { case FROM -> { - List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByTo != null && !relationsByTo.isEmpty()) { - EntityRelation entityRelation = relationsByTo.get(0); // only one supported + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byToAndType != null && !byToAndType.isEmpty()) { + EntityRelation entityRelation = byToAndType.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } } case TO -> { - List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByFrom != null && !relationsByFrom.isEmpty()) { - for (EntityRelation entityRelation : relationsByFrom) { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byFromAndType != null && !byFromAndType.isEmpty()) { + for (EntityRelation entityRelation : byFromAndType) { if (entityRelation.getTo().equals(cf.getEntityId())) { result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); } @@ -665,25 +605,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesTimeSeries(updatedTelemetry); - } else if (!proto.getAttrDataList().isEmpty()) { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); - } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); - } else { - return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); - } - } - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); log.debug("Received linked telemetry msg from entity [{}]", sourceEntityId); @@ -889,8 +810,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index db15fc1dc8..c30a12a8f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -49,13 +48,11 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -65,6 +62,7 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -123,30 +121,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { - RelationPathLevel relation = aggSource.getRelation(); + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - List>> relationListsFut = new ArrayList<>(); - if (aggSource.getEntityProfiles().isEmpty()) { - relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); - } else { - aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); - } - - return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { - if (relationLists == null) { + return Futures.transform(relationsFut, relations -> { + if (relations == null) { return new ArrayList<>(); } - List allRelations = relationLists.stream() - .filter(Objects::nonNull) - .flatMap(List::stream) - .toList(); return switch (relation.direction()) { - case FROM -> allRelations.stream() + case FROM -> relations.stream() .map(EntityRelation::getTo) .toList(); - case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); }; }, calculatedFieldCallbackExecutor); } @@ -193,28 +180,26 @@ public abstract class AbstractCalculatedFieldProcessingService { protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); - }); - return futures; + return aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + )); } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); - futures.put(key, argEntryFut); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), + + Map> argsFutures = aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) + )); + + return Futures.whenAllComplete(argsFutures.values()) + .call(() -> resolveArgumentFutures(argsFutures), MoreExecutors.directExecutor()); } @@ -347,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - return new AggSingleArgumentEntry(entityId, attributeKvEntry); + return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -359,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); - return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index e32ca42f9c..27e989de70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); - List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ae8238fb8a..918d71e326 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -43,8 +42,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,7 +49,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; -import java.util.stream.Collectors; @Service @Slf4j @@ -73,7 +69,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); + private final ConcurrentMap aggCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -87,8 +83,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cf); } } }); @@ -156,13 +152,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) - .map(Map.Entry::getKey) + public List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { + return aggCalculatedFields.keySet().stream() .map(this::getCalculatedFieldCtx) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .filter(relatedEntityFilter) + .toList(); } @Override @@ -206,8 +200,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -240,7 +234,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); - cfTriggers.remove(calculatedFieldId); + aggCalculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 44cb2aaee4..37c7b2ef6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,8 +27,6 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -89,7 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), - cfTrigger -> cfTrigger.matchesTimeSeries(entries), + cf -> cf.relatedEntityMatches(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -108,7 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), - cfTrigger -> cfTrigger.matchesAttributes(entries, scope), + cf -> cf.relatedEntityMatches(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -126,7 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), - cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), + cf -> cf.matchesRelatedEntityKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -138,7 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), - cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), + cf -> cf.matchesRelatedEntityKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -146,12 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, - Predicate cfAggKeysFilter, + Predicate relatedEntityFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, relatedEntityFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -162,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate relatedEntityFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -189,26 +187,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - AggSource source = aggConfig.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = aggConfig.getRelation(); switch (relation.direction()) { case FROM -> { List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byToAndType.isEmpty()) { return true; } } case TO -> { List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byFromAndType.isEmpty()) { return true; } @@ -307,10 +298,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } - private boolean isProfileEntity(EntityType entityType) { - return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); - } - private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f22b10a9c1..4e3d00ee62 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -74,4 +74,8 @@ public interface ArgumentEntry { return new AggArgumentEntry(entityIdkvEntryMap, false); } + static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { + return new AggSingleArgumentEntry(entityId, kvEntry); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 963ec3f06f..f8b9f5b665 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -83,8 +83,7 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; - private final List aggInputs; - private final Map aggregationInputs; + private final Map relatedEntityArguments; private final List argNames; private Output output; private String expression; @@ -123,11 +122,10 @@ public class CalculatedFieldCtx { this.mainEntityArguments = new HashMap<>(); this.linkedEntityArguments = new HashMap<>(); this.dynamicEntityArguments = new HashMap<>(); + this.relatedEntityArguments = new HashMap<>(); this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); - this.aggInputs = new ArrayList<>(); - this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -135,6 +133,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + relatedEntityArguments.put(refKey, entry.getKey()); + continue; + } if (entry.getValue().hasRelationQuerySource()) { relationQueryDynamicArguments = true; continue; @@ -172,9 +174,6 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - aggInputs.addAll(aggConfig.getInputs().values()); - aggregationInputs.putAll(aggConfig.getInputs()); - this.argNames.addAll(aggConfig.getInputs().keySet()); this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); } this.systemContext = systemContext; @@ -475,41 +474,57 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeriesKeys(map, keys); } - public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean relatedEntityMatches(List values) { + return matchesTimeSeries(relatedEntityArguments, values); + } + + public boolean relatedEntityMatches(List values, AttributeScope scope) { + return matchesAttributes(relatedEntityArguments, values, scope); + } + + public boolean matchesRelatedEntityKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(relatedEntityArguments, keys, scope); + } + + public boolean matchesRelatedEntityKeys(List keys) { + return matchesTimeSeriesKeys(relatedEntityArguments, keys); + } + + public boolean relatedEntityMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry); + return relatedEntityMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry, scope); + return relatedEntityMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); + return matchesRelatedEntityKeys(proto.getRemovedTsKeysList()); } else { - return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesRelatedEntityKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } - public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggTimeSeries(updatedTelemetry); + return dynamicSourceMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggAttributes(updatedTelemetry, scope); + return dynamicSourceMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesAggKeys(proto.getRemovedTsKeysList()); + return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); } else { - return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } @@ -544,67 +559,6 @@ public class CalculatedFieldCtx { return argNames; } - public boolean matchesAggKeys(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (String key : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggTimeSeries(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (TsKvEntry tsKvEntry : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (String key : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributes(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (AttributeKvEntry attributeKvEntry : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -662,7 +616,7 @@ public class CalculatedFieldCtx { private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json new file mode 100644 index 0000000000..32cd053d41 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -0,0 +1,60 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET_PROFILE", + "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "arguments": { + "oc": { + "refEntityKey": { + "key": "occupied", + "type": "TS_LATEST" + }, + "defaultValue": "false" + } + }, + "deduplicationIntervalMillis": 20000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6f6fabbea2..6485508602 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,6 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -54,6 +55,14 @@ public class CalculatedFieldArgumentUtils { } } + public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); + } else { + return new AggSingleArgumentEntry(); + } + } + public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 2ed3554666..cd9c1c578b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -27,8 +27,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -36,7 +36,6 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; @@ -53,9 +52,7 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -77,8 +74,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private CalculatedField calculatedField; - private long deduplicationInterval = 10000; @Before @@ -111,10 +106,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device1.getId(), "Contains"); createEntityRelation(asset.getId(), device2.getId(), "Contains"); - - calculatedField = createOccupancyCF("Occupied spaces", asset.getId(), List.of(deviceProfile.getId())); - - checkInitialCalculation(); } @After @@ -124,8 +115,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteTenant(savedTenant.getId()); } + @Test + public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF("Occupied spaces", asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + @Test public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -141,6 +157,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll @Test public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) @@ -161,35 +180,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testChangeProfile_checkMetricsCalculation() throws Exception { - DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); - device1.setDeviceProfileId(deviceProfile2.getId()); - device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + public void testCreateRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - postTelemetry(device1.getId(), "{\"occupied\":false}"); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); }); } @Test - public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - - postTelemetry(device3.getId(), "{\"occupied\":true}"); - - await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(this::checkInitialCalculationValues); + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -197,8 +214,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); }); } @@ -213,22 +230,22 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - CalculatedField calculatedField2 = createOccupancyCF("Occupied spaces 2", assetProfile.getId(), List.of(deviceProfile.getId())); + createOccupancyCF("Occupied spaces 2", assetProfile.getId()); await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset1).isNotNull(); + assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset2).isNotNull(); + assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); postTelemetry(device3.getId(), "{\"occupied\":true}"); @@ -243,23 +260,26 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); } - - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); - }); - } - +// +// @Test +// public void testChangeProfile_checkMetricsCalculation() throws Exception { +// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); +// device1.setDeviceProfileId(deviceProfile2.getId()); +// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); +// +// postTelemetry(device1.getId(), "{\"occupied\":false}"); +// +// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); +// }); +// } +// // @Test // public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { // Device device3 = createDevice("Device 3", "1234567890333"); @@ -296,18 +316,24 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); freeSpaces.setFunction(AggFunction.COUNT); - freeSpaces.setFilter("return oc == false"); + freeSpaces.setFilter("return oc == false;"); freeSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("freeSpaces", freeSpaces); AggMetric occupiedSpaces = new AggMetric(); occupiedSpaces.setFunction(AggFunction.COUNT); - occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setFilter("return oc == true;"); occupiedSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("occupiedSpaces", occupiedSpaces); @@ -320,23 +346,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output.setType(OutputType.TIME_SERIES); return createAggCf(name, entityId, - buildSource(EntitySearchDirection.FROM, "Contains", profiles), - Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, aggMetrics, output); } - private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { - AggSource source = new AggSource(); - source.setRelation(new RelationPathLevel(direction, relationType)); - source.setEntityProfiles(profiles); - return source; - } - private CalculatedField createAggCf(String name, EntityId entityId, - AggSource aggSource, - Map inputs, + RelationPathLevel relation, + Map inputs, Map metrics, Output output) { CalculatedField calculatedField = new CalculatedField(); @@ -345,8 +364,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); - configuration.setSource(aggSource); - configuration.setInputs(inputs); + configuration.setRelation(relation); + configuration.setArguments(inputs); configuration.setDeduplicationIntervalMillis(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java deleted file mode 100644 index 65d545bc6e..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Builder; -import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; -import java.util.function.Predicate; - -@Data -@Builder -public class CfAggTrigger { - - private List entityProfiles; - private List inputs; - - public boolean matches(EntityId profileId, Predicate cfAggTrigger) { - if (matchesProfile(profileId)) { - return cfAggTrigger.test(this); - } - return false; - } - - public boolean matchesProfile(EntityId profileId) { - return entityProfiles.isEmpty() || entityProfiles.contains(profileId); - } - - public boolean matchesTimeSeries(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - for (TsKvEntry tsKvEntry : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesAttributes(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - for (AttributeKvEntry attributeKvEntry : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesTimeSeriesKeys(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - - for (String key : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - - for (String key : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 2760e1855b..89f7e9a339 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -17,18 +17,18 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.relation.RelationPathLevel; -import java.util.List; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { +public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { - private AggSource source; - private Map inputs; + private RelationPathLevel relation; + private Map arguments; private long deduplicationIntervalMillis; private Map metrics; private Output output; @@ -42,11 +42,4 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public void validate() { } - public CfAggTrigger buildTrigger() { - return CfAggTrigger.builder() - .inputs(List.copyOf(inputs.values())) - .entityProfiles(List.copyOf(source.getEntityProfiles())) - .build(); - } - } From 4137a4a615ef4f8cfc84bb82878661feba9db4dd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 10:05:48 +0300 Subject: [PATCH 044/122] refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 105 ++++++++----- ...alculatedFieldManagerMessageProcessor.java | 139 ++++++------------ .../CalculatedFieldRelatedEntityMsg.java | 9 +- 3 files changed, 117 insertions(+), 136 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index e253633727..63aaf2bd68 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,7 +171,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } } + } + } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -202,63 +203,91 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - EntityId msgEntityId = msg.getEntityId(); - if (msgEntityId instanceof CalculatedFieldId cfId) { - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); - } else { - msg.getCallback().onSuccess(); - } + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } else { - if (states.isEmpty()) { - msg.getCallback().onSuccess(); - } - for (Map.Entry entry : states.entrySet()) { - LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); - state.getArguments().forEach((argName, argEntry) -> { - AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; - aggArgEntry.getAggInputs().remove(msgEntityId); - }); - state.getInputs().remove(msgEntityId); - state.setLastMetricsEvalTs(-1); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); - } + msg.getCallback().onSuccess(); } } } public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); - CalculatedFieldCtx cfCtx = msg.getCalculatedField(); - var state = states.get(cfCtx.getCfId()); - Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + log.debug("[{}] Processing CF {} related entity msg.", msg.getRelatedEntityId(), msg.getAction()); + switch (msg.getAction()) { + case UPDATED -> handleRelationUpdate(msg); + case DELETED -> handleRelationDelete(msg); + default -> msg.getCallback().onSuccess(); + } + } + + private void handleRelationUpdate(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + var state = states.get(ctx.getCfId()); try { + boolean justRestored = false; if (state == null) { - state = createState(cfCtx); - } else { - state.setCtx(cfCtx, actorCtx); + state = createState(ctx); + justRestored = true; } if (state.isSizeOk()) { - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + Map updatedArgs = new HashMap<>(); + if (!justRestored) { + updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); } - state.update(fetchedArguments, cfCtx); - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); - states.put(cfCtx.getCfId(), state); - processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); } else { - throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } } catch (Exception e) { - log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); if (e instanceof CalculatedFieldException cfe) { throw cfe; } - throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { + Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); + Map updatedArgs = state.update(fetchedArgs, ctx); + + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + return updatedArgs; + } + + private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + CalculatedFieldId cfId = ctx.getCfId(); + CalculatedFieldState state = states.get(cfId); + if (state == null) { + msg.getCallback().onSuccess(); + return; + } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + cleanupAggregationState(msg.getRelatedEntityId(), aggState); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } else { + msg.getCallback().onSuccess(); } } + private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + state.getArguments().values().forEach(argEntry -> { + AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + state.getInputs().remove(relatedEntityId); + state.setLastMetricsEvalTs(-1); + } + @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 620eee41ee..37ec4cc3d1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.TriConsumer; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -303,56 +305,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - }); - - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); - - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - }); - + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); } catch (Exception e) { callback.onSuccess(); } @@ -360,59 +320,43 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); - }); + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); + } catch (Exception e) { + callback.onSuccess(); + } + } - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); + private void processRelationByDirection(EntitySearchDirection direction, + String relationType, + EntityId mainId, + MultipleTbCallback parentCallback, + TriConsumer relationAction) { + List cfsByEntityIdAndProfile = getCalculatedFieldsByEntityIdAndProfile(mainId); + if (cfsByEntityIdAndProfile.isEmpty()) { + parentCallback.onSuccess(); + return; + } - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); - }); + List matchingCfs = cfsByEntityIdAndProfile.stream() + .filter(cf -> { + var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + RelationPathLevel relation = config.getRelation(); + return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); + }) + .toList(); + MultipleTbCallback directionCallback = new MultipleTbCallback(matchingCfs.size(), parentCallback); - } catch (Exception e) { - callback.onSuccess(); - } + matchingCfs.forEach(ctx -> + applyToTargetCfEntityActors(ctx, directionCallback, (entityId, cb) -> relationAction.accept(entityId, ctx, cb)) + ); } private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { @@ -629,9 +573,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = msg.getEntityId(); log.debug("Received changed owner msg from entity [{}]", entityId); updateEntityOwner(entityId); - List cfs = new ArrayList<>(); - cfs.addAll(getCalculatedFieldsByEntityId(entityId)); - cfs.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + List cfs = getCalculatedFieldsByEntityIdAndProfile(entityId); if (cfs.isEmpty()) { msgCallback.onSuccess(); return; @@ -695,6 +637,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { + List cfsByEntityIdAndProfile = new ArrayList<>(); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + return cfsByEntityIdAndProfile; + } + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { if (entityId == null) { return Collections.emptyList(); @@ -722,14 +671,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } - private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.DELETED, cf, callback)); } private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.UPDATED, cf, callback)); } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java index d3bdd9cc5c..bf73a1b1b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.Data; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; @@ -27,16 +28,18 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; - private final EntityId entityId; + private final EntityId relatedEntityId; + private final ActionType action; private final CalculatedFieldCtx calculatedField; private final TbCallback callback; public CalculatedFieldRelatedEntityMsg(TenantId tenantId, - EntityId entityId, + EntityId relatedEntityId, ActionType action, CalculatedFieldCtx calculatedField, TbCallback callback) { this.tenantId = tenantId; - this.entityId = entityId; + this.relatedEntityId = relatedEntityId; + this.action = action; this.calculatedField = calculatedField; this.callback = callback; } From 89753ed8ea492b49dd994adec9738a72714078a3 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 15 Oct 2025 16:13:38 +0300 Subject: [PATCH 045/122] CF: make type+name unique instead of just name --- .../src/main/data/upgrade/basic/schema_update.sql | 7 +++++++ .../cf/ctx/state/alarm/AlarmCalculatedFieldState.java | 2 ++ .../processor/cf/BaseCalculatedFieldProcessor.java | 2 +- .../server/dao/cf/CalculatedFieldService.java | 2 +- .../server/dao/cf/BaseCalculatedFieldService.java | 11 ++++++----- .../thingsboard/server/dao/cf/CalculatedFieldDao.java | 2 +- .../server/dao/sql/cf/CalculatedFieldRepository.java | 3 ++- .../server/dao/sql/cf/JpaCalculatedFieldDao.java | 4 ++-- dao/src/main/resources/sql/schema-entities.sql | 2 +- .../dao/service/CalculatedFieldServiceTest.java | 2 +- 10 files changed, 24 insertions(+), 13 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 0add4c0545..495aee00e2 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -46,3 +46,10 @@ WHERE NOT ( ); -- UPDATE TENANT PROFILE CONFIGURATION END + +-- CALCULATED FIELD UNIQUE CONSTRAINT UPDATE START + +ALTER TABLE calculated_field DROP CONSTRAINT IF EXISTS calculated_field_unq_key; +ALTER TABLE calculated_field ADD CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, type, name); + +-- CALCULATED FIELD UNIQUE CONSTRAINT UPDATE END diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 657fa80f63..c140ba78af 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -189,6 +189,8 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { initCurrentAlarm(ctx); + // FIXME: don't create alarm if attrs were deleted, or config is updated + // TODO: what if expression is changed? do we reevaluate? or only on new events? TbAlarmResult result = createOrClearAlarms(state -> { if (updatedArgs != null) { boolean newEvent = !updatedArgs.isEmpty(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/BaseCalculatedFieldProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/BaseCalculatedFieldProcessor.java index 4ef6ec7ba2..c3f9300e24 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/BaseCalculatedFieldProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/BaseCalculatedFieldProcessor.java @@ -53,7 +53,7 @@ public abstract class BaseCalculatedFieldProcessor extends BaseEdgeProcessor { } String calculatedFieldName = calculatedField.getName(); - CalculatedField calculatedFieldByName = edgeCtx.getCalculatedFieldService().findByEntityIdAndName(calculatedField.getEntityId(), calculatedFieldName); + CalculatedField calculatedFieldByName = edgeCtx.getCalculatedFieldService().findByEntityIdAndTypeAndName(calculatedField.getEntityId(), calculatedField.getType(), calculatedFieldName); if (calculatedFieldByName != null && !calculatedFieldByName.getId().equals(calculatedFieldId)) { calculatedFieldName = calculatedFieldName + "_" + StringUtils.randomAlphabetic(15); log.warn("[{}] calculatedField with name {} already exists. Renaming calculatedField name to {}", diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e0481b6705..57c6df3c7f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -36,7 +36,7 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - CalculatedField findByEntityIdAndName(EntityId entityId, String name); + CalculatedField findByEntityIdAndTypeAndName(EntityId entityId, CalculatedFieldType type, String name); List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 2efa32215a..6b46d3dfb3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -89,8 +89,9 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, - "calculated_field_unq_key", "Calculated Field with such name is already in exists!", - "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + "calculated_field_unq_key", calculatedField.getType() == CalculatedFieldType.ALARM ? + "Alarm rule with such type already exists" : "Calculated field with such name and type already exists", + "calculated_field_external_id_unq_key", "Calculated field with such external id already exists"); throw e; } } @@ -104,10 +105,10 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } @Override - public CalculatedField findByEntityIdAndName(EntityId entityId, String name) { - log.trace("Executing findByEntityIdAndName [{}], calculatedFieldName[{}]", entityId, name); + public CalculatedField findByEntityIdAndTypeAndName(EntityId entityId, CalculatedFieldType type, String name) { + log.trace("Executing findByEntityIdAndTypeAndName entityId [{}], type [{}], name [{}]", entityId, type, name); validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); - return calculatedFieldDao.findByEntityIdAndName(entityId, name); + return calculatedFieldDao.findByEntityIdAndTypeAndName(entityId, type, name); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 40517fe78b..21b63e8d27 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -37,7 +37,7 @@ public interface CalculatedFieldDao extends Dao { List findAll(); - CalculatedField findByEntityIdAndName(EntityId entityId, String name); + CalculatedField findByEntityIdAndTypeAndName(EntityId entityId, CalculatedFieldType type, String name); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index f2c3525a4f..9a1f904788 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -19,6 +19,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -29,7 +30,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 7a8c914e74..e0e5ef60c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -68,8 +68,8 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao calculatedFieldService.save(calculatedField)) .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field with such name is already in exists!"); + .hasMessage("Calculated field with such name and type already exists"); } @Test From a693cabc05de347138d9116ce01b3c0593543889 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 16:44:36 +0300 Subject: [PATCH 046/122] added more tests --- ...CalculatedFieldEntityMessageProcessor.java | 46 ++- ...alculatedFieldManagerMessageProcessor.java | 6 - ...tractCalculatedFieldProcessingService.java | 3 +- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../ctx/state/BaseCalculatedFieldState.java | 11 + .../cf/ctx/state/CalculatedFieldCtx.java | 6 +- .../ctx/state/SimpleCalculatedFieldState.java | 10 - .../ctx/state/SingleValueArgumentEntry.java | 9 + .../state/aggregation/AggArgumentEntry.java | 8 +- ...java => AggSingleEntityArgumentEntry.java} | 17 +- ...ValuesAggregationCalculatedFieldState.java | 103 ++++-- .../aggregation/function/AvgAggEntry.java | 2 +- .../state/aggregation/function/new_agg.json | 4 +- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 10 +- ...tValuesAggregationCalculatedFieldTest.java | 344 ++++++++++++------ ...gregationCalculatedFieldConfiguration.java | 1 + 17 files changed, 388 insertions(+), 202 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggSingleArgumentEntry.java => AggSingleEntityArgumentEntry.java} (79%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 63aaf2bd68..ce29dc06e0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -53,7 +53,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -67,6 +67,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -329,7 +330,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else if (proto.getAttrDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedTsKeysCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, msg.getEntityId(), proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedAttrKeysCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else { @@ -405,7 +406,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, entityId, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { @@ -565,7 +566,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(originator, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } } } @@ -618,7 +619,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } } } @@ -631,14 +632,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), geofencingArgumentNames, scope, removedAttrKeys); + List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, List geofencingArgNames, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, + Map argNames, + Map configArguments, + List geofencingArgNames, + List relatedEntityArgNames, + AttributeScopeProto scope, + List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); @@ -652,22 +660,36 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } Argument argument = configArguments.get(argName); String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); + : new SingleValueArgumentEntry(); + if (relatedEntityArgNames.contains(argName)) { + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + continue; + } + arguments.put(argName, argumentEntry); } return arguments; } - private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, EntityId entityId, List removedTelemetryKeys) { Map deletedArguments = ctx.getArguments().entrySet().stream() .filter(entry -> removedTelemetryKeys.contains(entry.getValue().getRefEntityKey().getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + fetchedArgs = fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) + )); + } else { + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + } + return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 37ec4cc3d1..af5a672157 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -516,12 +516,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId) { - return aggCalculatedFields.values().stream() - .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) - .toList(); - } - private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index c30a12a8f3..94695bcc8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -250,7 +250,8 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 4e3d00ee62..ca0a12e1d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -39,7 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), - @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -75,7 +75,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleArgumentEntry(entityId, kvEntry); + return new AggSingleEntityArgumentEntry(entityId, kvEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index f75711a107..2d853fd3fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Getter; import lombok.Setter; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -134,4 +135,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } + protected Object formatResult(double result, Integer decimals) { + if (decimals == null) { + return result; + } + if (decimals.equals(0)) { + return TbUtils.toInt(result); + } + return TbUtils.toFixed(result, decimals); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f8b9f5b665..3a9d75b3d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -107,6 +107,7 @@ public class CalculatedFieldCtx { private boolean relationQueryDynamicArguments; private List mainEntityGeofencingArgumentNames; private List linkedEntityAndCurrentOwnerGeofencingArgumentNames; + private List relatedEntityArgumentNames; private long scheduledUpdateIntervalMillis; @@ -126,6 +127,7 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.relatedEntityArgumentNames = new ArrayList<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -153,6 +155,7 @@ public class CalculatedFieldCtx { } } this.argNames.addAll(arguments.keySet()); + this.relatedEntityArgumentNames.addAll(relatedEntityArguments.values()); if (argBasedConfig instanceof ExpressionBasedCalculatedFieldConfiguration expressionBasedConfig) { this.expression = expressionBasedConfig.getExpression(); this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) argBasedConfig).isUseLatestTs(); @@ -174,7 +177,7 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); @@ -578,6 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 65cb595632..8886482ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -62,16 +62,6 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build()); } - private Object formatResult(double expressionResult, Integer decimals) { - if (decimals == null) { - return expressionResult; - } - if (decimals.equals(0)) { - return TbUtils.toInt(expressionResult); - } - return TbUtils.toFixed(expressionResult, decimals); - } - private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); if (result instanceof Double doubleValue) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 288b486e83..e81201c961 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -45,6 +45,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + this.ts = singleValueArgumentEntry.ts; + this.kvEntryValue = singleValueArgumentEntry.kvEntryValue; + this.version = singleValueArgumentEntry.version; + this.forceResetPrevious = singleValueArgumentEntry.forceResetPrevious; + } + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 12ae2c4638..7e3a8623e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -48,11 +48,11 @@ public class AggArgumentEntry implements ArgumentEntry { if (entry instanceof AggArgumentEntry aggArgumentEntry) { aggInputs.putAll(aggArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { - if (aggSingleArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { + if (aggSingleEntityArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6b81d5380c..32ce77311d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -30,34 +30,39 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; @Data @NoArgsConstructor @AllArgsConstructor -public class AggSingleArgumentEntry extends SingleValueArgumentEntry { +public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; private boolean deleted; - public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { super(ts, kvEntryValue, version); this.entityId = entityId; } @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { if (singleValueEntry.getTs() <= ts) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index d25cde020c..1df27b4cbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; @@ -43,10 +45,11 @@ import java.util.Map; import java.util.Map.Entry; @Slf4j -@Data +@Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { private long lastArgsRefreshTs = -1; + @Setter private long lastMetricsEvalTs = -1; private long deduplicationInterval = -1; private Map metrics; @@ -76,8 +79,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public void init() { super.init(); -// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); -// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); } @Override @@ -102,41 +104,51 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + if (!shouldRecalculate()) { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } + + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; - if (intervalPassed && argsUpdatedDuringInterval) { - ObjectNode aggResult = JacksonUtil.newObjectNode(); - for (Entry entry : metrics.entrySet()) { - String metricKey = entry.getKey(); - AggMetric metric = entry.getValue(); - - AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); - - for (Map entityInputs : inputs.values()) { - if (applyAggregation(metric.getFilter(), entityInputs)) { - Object arg = resolveAggregationInput(metric.getInput(), entityInputs); - if (arg != null) { - aggMetric.update(arg); - } - } - } + return intervalPassed && argsUpdatedDuringInterval; + } + + private ObjectNode aggregateMetrics(Output output) throws Exception { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); - aggMetric.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(result)); - }); + AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + aggregateMetric(metric, aggMetricEntry); + aggMetricEntry.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + }); + } + return aggResult; + } + + private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggEntry.update(arg); + } } - Output output = ctx.getOutput(); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(aggResult) - .build()); - } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); } } @@ -158,4 +170,25 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF } } + private Object formatResult(Object aggregationResult, Integer decimals) { + try { + double result = Double.parseDouble(aggregationResult.toString()); + return formatResult(result, decimals); + } catch (Exception e) { + throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); + } + } + + protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { + long latestTs = getLatestTimestamp(); + if (useLatestTs && latestTs != -1) { + ObjectNode resultNode = JacksonUtil.newObjectNode(); + resultNode.put("ts", latestTs); + resultNode.set("values", result); + return resultNode; + } else { + return result; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index ad1f2ee8a8..afe6abb93e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -35,7 +35,7 @@ public class AvgAggEntry extends BaseAggEntry { @Override protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json index 32cd053d41..c6b841b673 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET_PROFILE", - "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + "entityType": "ASSET", + "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6485508602..74cdb0cddb 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -59,7 +59,7 @@ public class CalculatedFieldArgumentUtils { if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); } else { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d4ee7bb2e3..19ad4bfb7c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,7 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; @@ -236,7 +236,7 @@ public class CalculatedFieldUtils { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; Map> arguments = new HashMap<>(); proto.getAggArgumentsList().forEach(argProto -> { - AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { @@ -248,14 +248,14 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { if (!proto.hasValue()) { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); SingleValueArgumentProto singleValueArgument = proto.getValue(); TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleArgumentEntry( + return new AggSingleEntityArgumentEntry( entityId, tsValueProto.getTs(), (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index cd9c1c578b..a4b43e362b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; @DaoSqlTest @@ -116,7 +117,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + public void testCreateCfOnProfile_checkInitialAggregation() throws Exception { Asset asset2 = createAsset("Asset 2", assetProfile.getId()); Device device3 = createDevice("Device 3", "1234567890333"); Device device4 = createDevice("Device 4", "1234567890444"); @@ -124,22 +125,153 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - createOccupancyCF("Occupied spaces", asset2.getId()); + createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testAddEntityToProfile_checkAggregation() throws Exception { + createOccupancyCF(assetProfile.getId()); + + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); + + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("freeSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("totalSpaces").get(0).get("value").isNull()).isTrue(); + }); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testChangeEntityProfile_checkAggregation() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(assetProfile.getId()); + + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + AssetProfile newAssetProfile = createAssetProfile("New Asset Profile"); + asset2.setAssetProfileId(newAssetProfile.getId()); + doPost("/api/asset", asset2, Asset.class); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfOnAssetAndNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfAndUpdateTelemetry_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -147,17 +279,38 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testDeleteCf_checkNoAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + doDelete("/api/calculatedField/" + cf.getId().getId().toString()) + .andExpect(status().isOk()); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateTelemetry_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -171,136 +324,92 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCreateRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); + public void testDeleteTelemetry_checkAggregationWithPreviousValuesOrDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":false}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + createOccupancyCF(asset2.getId()); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); }); - } - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCfOnProfile_checkMetricsCalculation() throws Exception { - Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + public void testCreateRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - Device device4 = createDevice("Device 4", deviceProfile.getId(), "1234567890444"); - postTelemetry(device4.getId(), "{\"occupied\":false}"); - createEntityRelation(asset2.getId(), device3.getId(), "Contains"); - createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); - createOccupancyCF("Occupied spaces 2", assetProfile.getId()); + createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset1).isNotNull(); - assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset2).isNotNull(); - assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "2", + "totalSpaces", "3" + )); }); + } - postTelemetry(device3.getId(), "{\"occupied\":true}"); + @Test + public void testDeleteRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "0", + "totalSpaces", "1" + )); }); } -// -// @Test -// public void testChangeProfile_checkMetricsCalculation() throws Exception { -// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); -// device1.setDeviceProfileId(deviceProfile2.getId()); -// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); -// -// postTelemetry(device1.getId(), "{\"occupied\":false}"); -// -// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); -// }); -// } -// -// @Test -// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { -// Device device3 = createDevice("Device 3", "1234567890333"); -// postTelemetry(device3.getId(), "{\"occupied\":true}"); -// createEntityRelation(asset.getId(), device3.getId(), "Contains"); -// -// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); -// configuration.getSource().setEntityProfiles(Collections.emptyList()); -// calculatedField.setConfiguration(configuration); -// saveCalculatedField(calculatedField); -// -// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); -// }); -// } private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -316,7 +425,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId) { + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); @@ -344,8 +453,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); - return createAggCf(name, entityId, + return createAggCf("Occupied spaces", entityId, new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), arguments, aggMetrics, @@ -393,6 +503,12 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doPost("/api/asset", asset, Asset.class); } + private void verifyTelemetry(EntityId entityId, Map expectedResults) throws Exception { + ObjectNode result = getLatestTelemetry(entityId, expectedResults.keySet().toArray(new String[0])); + assertThat(result).isNotNull(); + expectedResults.forEach((key, value) -> assertThat(result.get(key).get(0).get("value").asText()).isEqualTo(value)); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 89f7e9a339..43a3360ce6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -32,6 +32,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private long deduplicationIntervalMillis; private Map metrics; private Output output; + private boolean useLatestTs; @Override public CalculatedFieldType getType() { From fca21661707023180d28bec8f6b213540f7e1e92 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 16 Oct 2025 11:53:59 +0300 Subject: [PATCH 047/122] UI: Add configuration propagate calculate field --- .../calculated-field.module.ts | 6 +- .../calculated-fields-table-config.ts | 70 ++++--- ...ulated-field-argument-panel.component.html | 134 +++++++++----- ...ulated-field-argument-panel.component.scss | 0 ...lculated-field-argument-panel.component.ts | 113 ++++++++---- ...lated-field-arguments-table.component.html | 15 +- ...lated-field-arguments-table.component.scss | 2 +- ...culated-field-arguments-table.component.ts | 58 +++--- ...calculated-field-arguments-table.module.ts | 45 +++++ .../propagate-arguments-table.component.ts | 116 ++++++++++++ .../calculated-field-dialog.component.html | 10 +- .../calculated-field-dialog.component.ts | 11 ++ .../geofencing-configuration.component.ts | 2 +- .../geofencing-configuration.module.ts | 2 +- ...e.ts => calculated-field-output.module.ts} | 0 .../propagation-configuration.component.html | 99 ++++++++++ .../propagation-configuration.component.ts | 174 ++++++++++++++++++ .../propagation-configuration.module.ts | 44 +++++ .../simple-configuration.component.html | 2 +- .../simple-configuration.component.ts | 21 ++- .../simple-configuration.module.ts | 12 +- .../shared/models/calculated-field.models.ts | 51 ++++- .../assets/locale/locale.constant-en_US.json | 23 ++- 23 files changed, 828 insertions(+), 182 deletions(-) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-argument-panel.component.html (65%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-argument-panel.component.scss (100%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-argument-panel.component.ts (76%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-arguments-table.component.html (89%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-arguments-table.component.scss (95%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{simple-configuration => calculated-field-arguments}/calculated-field-arguments-table.component.ts (85%) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts rename ui-ngx/src/app/modules/home/components/calculated-fields/components/output/{caclculate-field-output.module.ts => calculated-field-output.module.ts} (100%) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index 9cccf28305..6cb6a907b7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -40,6 +40,9 @@ import { import { SimpleConfigurationModule } from '@home/components/calculated-fields/components/simple-configuration/simple-configuration.module'; +import { + PropagationConfigurationModule +} from '@home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module'; @NgModule({ declarations: [ @@ -55,7 +58,8 @@ import { GeofencingConfigurationModule, EntityDebugSettingsButtonComponent, HomeComponentsModule, - SimpleConfigurationModule + SimpleConfigurationModule, + PropagationConfigurationModule, ], exports: [ CalculatedFieldsTableComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 1105365347..4104b97221 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -40,10 +40,12 @@ import { ArgumentType, CalculatedField, CalculatedFieldEventArguments, + CalculatedFieldScriptConfiguration, CalculatedFieldType, CalculatedFieldTypeTranslations, getCalculatedFieldArgumentsEditorCompleter, getCalculatedFieldArgumentsHighlights, + PropagationWithExpression, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -122,7 +124,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('createdTime', 'common.created-time', this.datePipe, '150px')); this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); - this.columns.push(new EntityTableColumn('type', 'common.type', '70px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(new EntityTableColumn('type', 'common.type', '80px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); this.columns.push(expressionColumn); this.cellActionDescriptors.push( @@ -156,7 +158,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { - if (calculatedField.type === CalculatedFieldType.GEOFENCING || calculatedField.type === CalculatedFieldType.SIMPLE) { + if ( + calculatedField.type === CalculatedFieldType.SCRIPT || + (calculatedField.type === CalculatedFieldType.PROPAGATION && calculatedField.configuration.applyExpressionToResolvedArguments === true) + ) { + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + const type = calculatedField.configuration.arguments[key].refEntityKey.type; + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) + ? {...argumentsObj[key], type} + : type === ArgumentType.Rolling ? {values: [], type} : {value: '', type, ts: new Date().getTime()}; + return acc; + }, {}); + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: resultArguments, + expression: (calculatedField.configuration as CalculatedFieldScriptConfiguration | PropagationWithExpression).expression, + argumentsEditorCompleter: getCalculatedFieldArgumentsEditorCompleter(calculatedField.configuration.arguments), + argumentsHighlightRules: getCalculatedFieldArgumentsHighlights(calculatedField.configuration.arguments), + openCalculatedFieldEdit + } + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({ + entityId: this.entityId, ...calculatedField, + configuration: {...calculatedField.configuration, expression} as any + }, true) + } + }), + ); + } else { return of(null); } - const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { - const type = calculatedField.configuration.arguments[key].refEntityKey.type; - acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) - ? { ...argumentsObj[key], type } - : type === ArgumentType.Rolling ? { values: [], type } : { value: '', type, ts: new Date().getTime() }; - return acc; - }, {}); - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, - { - disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], - data: { - arguments: resultArguments, - expression: calculatedField.configuration.expression, - argumentsEditorCompleter: getCalculatedFieldArgumentsEditorCompleter(calculatedField.configuration.arguments), - argumentsHighlightRules: getCalculatedFieldArgumentsHighlights(calculatedField.configuration.arguments), - openCalculatedFieldEdit - } - }).afterClosed() - .pipe( - filter(Boolean), - tap(expression => { - if (openCalculatedFieldEdit) { - this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) - } - }), - ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html similarity index 65% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 92855d882f..77bdedb068 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -19,62 +19,34 @@
{{ 'calculated-fields.argument-settings' | translate }}
-
-
{{ 'calculated-fields.argument-name' | translate }}
- - - @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('forbiddenName')) { - - warning - - } - -
- + @if (!isOutputKey) { + + } +
{{ 'entity.entity-type' | translate }}
- + @for (type of argumentEntityTypes; track type) { {{ ArgumentEntityTypeTranslations.get(type) | translate }} } + @if (argumentType.touched && argumentType.hasError('required')) { + + warning + + }
@if (ArgumentEntityTypeParamsMap.has(entityType)) { @@ -83,7 +55,8 @@ + @if (isOutputKey) { + + } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) {
{{ 'calculated-fields.default-value' | translate }}
@@ -207,3 +190,54 @@
+ + +
+
{{ label | translate }}
+ + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('forbiddenName')) { + + warning + + } + +
+ diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.scss similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.scss rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts similarity index 76% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index 8ccaa4fe49..070ffb5c06 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -14,7 +14,16 @@ /// limitations under the License. /// -import { AfterViewInit, ChangeDetectorRef, Component, Input, OnInit, output, ViewChild } from '@angular/core'; +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + Input, + OnInit, + output, + ViewChild +} from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; @@ -25,7 +34,6 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - CalculatedFieldType, getCalculatedFieldCurrentEntityFilter } from '@shared/models/calculated-field.models'; import { debounceTime, delay, distinctUntilChanged, filter } from 'rxjs/operators'; @@ -43,6 +51,7 @@ import { AppState } from '@core/core.state'; import { Store } from '@ngrx/store'; import { EntityAutocompleteComponent } from '@shared/components/entity/entity-autocomplete.component'; import { NULL_UUID } from '@shared/models/id/has-uuid'; +import { TenantId } from '@shared/models/id/tenant-id'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -56,22 +65,23 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; - @Input() calculatedFieldType: CalculatedFieldType; + @Input() isScript: boolean; @Input() usedArgumentNames: string[]; + @Input() isOutputKey = false; + @Input() argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; @ViewChild('entityAutocomplete') entityAutocomplete: EntityAutocompleteComponent; argumentsDataApplied = output(); + argumentType = this.fb.control(ArgumentEntityType.Current, Validators.required); + readonly maxDataPointsPerRollingArg = getCurrentAuthState(this.store).maxDataPointsPerRollingArg; readonly defaultLimit = Math.floor(this.maxDataPointsPerRollingArg / 10); argumentFormGroup = this.fb.group({ argumentName: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenArgumentNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], - refEntityId: this.fb.group({ - entityType: [ArgumentEntityType.Current], - id: [''] - }), + refEntityId: [null], refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], key: ['', [Validators.pattern(oneSpaceInsideRegex)]], @@ -86,7 +96,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI entityFilter: EntityFilter; entityNameSubject = new BehaviorSubject(null); - readonly argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; readonly ArgumentType = ArgumentType; readonly DataKeyType = DataKeyType; @@ -103,20 +112,17 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI private fb: FormBuilder, private cd: ChangeDetectorRef, private popover: TbPopoverComponent, - private store: Store + private store: Store, + private destroyRef: DestroyRef ) { this.observeEntityFilterChanges(); - this.observeEntityTypeChanges(); + this.observeArgumentTypeChanges(); this.observeEntityKeyChanges(); this.observeUpdatePosition(); } get entityType(): ArgumentEntityType { - return this.argumentFormGroup.get('refEntityId').get('entityType').value; - } - - get refEntityIdFormGroup(): FormGroup { - return this.argumentFormGroup.get('refEntityId') as FormGroup; + return this.argumentType.value; } get refEntityKeyFormGroup(): FormGroup { @@ -130,14 +136,18 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } ngOnInit(): void { + this.updatedArgumentType(); this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); - this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.updateEntityFilter(this.entityType, true); + this.updatedRefEntityIdState(this.entityType); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); + this.setInitialEntityType(); + this.setWatchKeyChange(); this.argumentTypes = Object.values(ArgumentType) - .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + .filter(type => type !== ArgumentType.Rolling || this.isScript); } ngAfterViewInit(): void { @@ -147,12 +157,11 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } saveArgument(): void { - const { refEntityId, ...restConfig } = this.argumentFormGroup.value; - const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; - if (refEntityId.entityType === ArgumentEntityType.Tenant) { - refEntityId.id = this.tenantId; + const value = this.argumentFormGroup.value as CalculatedFieldArgumentValue; + if (this.entityType === ArgumentEntityType.Tenant) { + value.refEntityId = new TenantId(this.tenantId) as any; } - if (refEntityId.entityType !== ArgumentEntityType.Current && refEntityId.entityType !== ArgumentEntityType.Tenant) { + if (this.entityType !== ArgumentEntityType.Current && this.entityType !== ArgumentEntityType.Tenant) { value.entityName = this.entityNameSubject.value; } if (value.defaultValue) { @@ -166,6 +175,14 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.popover.hide(); } + private updatedArgumentType(): void { + let argumentType = ArgumentEntityType.Current; + if (this.argument.refEntityId?.entityType) { + argumentType = this.argument.refEntityId.entityType; + } + this.argumentType.setValue(argumentType, {emitEvent: false}); + } + private toggleByEntityKeyType(type: ArgumentType): void { const isAttribute = type === ArgumentType.Attribute; const isRolling = type === ArgumentType.Rolling; @@ -205,26 +222,21 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI private observeEntityFilterChanges(): void { merge( - this.refEntityIdFormGroup.get('entityType').valueChanges, + this.argumentType.valueChanges, this.refEntityKeyFormGroup.get('type').valueChanges, - this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + this.argumentFormGroup.get('refEntityId').valueChanges.pipe(filter(Boolean)), this.refEntityKeyFormGroup.get('scope').valueChanges, ) .pipe(debounceTime(50), takeUntilDestroyed()) .subscribe(() => this.updateEntityFilter(this.entityType)); } - private observeEntityTypeChanges(): void { - this.refEntityIdFormGroup.get('entityType').valueChanges + private observeArgumentTypeChanges(): void { + this.argumentType.valueChanges .pipe(distinctUntilChanged(), takeUntilDestroyed()) .subscribe(type => { - this.argumentFormGroup.get('refEntityId').get('id').setValue(''); - const isEntityWithId = type !== ArgumentEntityType.Tenant && type !== ArgumentEntityType.Current; - this.argumentFormGroup.get('refEntityId') - .get('id')[isEntityWithId ? 'enable' : 'disable'](); - if (!isEntityWithId) { - this.entityNameSubject.next(null); - } + this.argumentFormGroup.get('refEntityId').setValue(null); + this.updatedRefEntityIdState(type); if (!this.enableAttributeScopeSelection) { this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); } @@ -247,29 +259,56 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } private setInitialEntityKeyType(): void { - if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + if (!this.isScript && this.argument.refEntityKey?.type === ArgumentType.Rolling) { const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); typeControl.setValue(null); typeControl.markAsTouched(); } } + private setInitialEntityType() { + if (!this.argumentEntityTypes.includes(this.entityType)) { + this.argumentType.setValue(null); + this.argumentType.markAsTouched(); + } + } + + private setWatchKeyChange(): void { + if (this.isOutputKey) { + this.refEntityKeyFormGroup.get('key').valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe((key) => { + if (this.argumentFormGroup.get('argumentName').pristine) { + this.argumentFormGroup.get('argumentName').setValue(key); + } + }); + } + } + private forbiddenArgumentNameValidator(): ValidatorFn { return (control: FormControl) => { const trimmedValue = control.value.trim().toLowerCase(); - const forbiddenArgumentNames = ['ctx', 'e', 'pi']; + const forbiddenArgumentNames = ['ctx', 'e', 'pi', 'propagationCtx']; return forbiddenArgumentNames.includes(trimmedValue) ? { forbiddenName: true } : null; }; } private observeUpdatePosition(): void { merge( - this.refEntityIdFormGroup.get('entityType').valueChanges, + this.argumentType.valueChanges, this.refEntityKeyFormGroup.get('type').valueChanges, this.argumentFormGroup.get('timeWindow').valueChanges, - this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + this.argumentFormGroup.get('refEntityId').valueChanges.pipe(filter(Boolean)), ) .pipe(delay(50), takeUntilDestroyed()) .subscribe(() => this.popover.updatePosition()); } + + private updatedRefEntityIdState(type: ArgumentEntityType): void { + const isEntityWithId = !!type && type !== ArgumentEntityType.Tenant && type !== ArgumentEntityType.Current; + this.argumentFormGroup.get('refEntityId')[isEntityWithId ? 'enable' : 'disable'](); + if (!isEntityWithId) { + this.entityNameSubject.next(null); + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.html similarity index 89% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.html index 8cf040538c..94e2967d23 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.html @@ -21,7 +21,7 @@ [matSortActive]="sortOrder.property" [matSortDirection]="sortOrder.direction" matSortDisableClear> -
{{ 'common.name' | translate }}
+
{{ argumentNameColumn | translate }}
@@ -29,7 +29,7 @@ @@ -37,7 +37,7 @@ - + {{ 'entity.entity-type' | translate }} @@ -96,8 +96,7 @@ [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> - - + +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.scss similarity index 95% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.scss rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.scss index 430958d0f4..6ddb58c51c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.scss @@ -62,7 +62,7 @@ } .arguments-table { - .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell:nth-child(2) { padding: 0 28px 0 0; } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.ts similarity index 85% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.ts index 03730f3c69..8187c360c1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component.ts @@ -45,17 +45,17 @@ import { } from '@shared/models/calculated-field.models'; import { CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component'; +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; -import { getEntityDetailsPageURL, isEqual } from '@core/utils'; +import { getEntityDetailsPageURL, isDefined, isEqual } from '@core/utils'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; import { EntityService } from '@core/http/entity.service'; -import { MatSort } from '@angular/material/sort'; +import { MatSort, SortDirection } from '@angular/material/sort'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -85,16 +85,22 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; - @Input() calculatedFieldType: CalculatedFieldType; + @Input() isScript: boolean; @ViewChild(MatSort, { static: true }) sort: MatSort; errorText = ''; argumentsFormArray = this.fb.array([]); entityNameMap = new Map(); - sortOrder = { direction: 'asc', property: '' }; + sortOrder: { direction: SortDirection; property: string } = {direction: 'asc', property: ''}; dataSource = new CalculatedFieldArgumentDatasource(); + argumentNameColumn = 'common.name'; + argumentNameColumnCopy = 'calculated-fields.copy-argument-name'; + displayColumns = ['name', 'entityType', 'target', 'type', 'key', 'actions']; + + protected panelAdditionalCtx: Record + readonly entityTypeTranslations = entityTypeTranslations; readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly ArgumentEntityType = ArgumentEntityType; @@ -107,14 +113,14 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( - private fb: FormBuilder, - private popoverService: TbPopoverService, - private viewContainerRef: ViewContainerRef, - private cd: ChangeDetectorRef, - private renderer: Renderer2, - private entityService: EntityService, - private destroyRef: DestroyRef, - private store: Store + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store ) { this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { this.updateDataSource(value); @@ -123,9 +129,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } ngOnChanges(changes: SimpleChanges): void { - if (changes.calculatedFieldType?.previousValue - && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { - this.argumentsFormArray.updateValueAndValidity(); + if (isDefined(changes.isScript?.previousValue) && changes.isScript.currentValue !== changes.isScript.previousValue) { + this.changeIsScriptMode(); } } @@ -141,7 +146,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.propagateChange = fn; } - registerOnTouched(_): void {} + registerOnTouched(_: any): void {} validate(): ValidationErrors | null { this.updateErrorText(); @@ -170,7 +175,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces index, argument, entityId: this.entityId, - calculatedFieldType: this.calculatedFieldType, + isScript: this.isScript, buttonTitle: isExists ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, @@ -181,8 +186,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces renderer: this.renderer, componentType: CalculatedFieldArgumentPanelComponent, hostView: this.viewContainerRef, - preferredPlacement: isExists ? ['left', 'leftTop', 'leftBottom'] : ['topRight', 'right', 'rightTop'], - context: ctx, + preferredPlacement: isExists ? ['leftOnly', 'leftTopOnly', 'leftBottomOnly'] : ['rightOnly', 'rightTopOnly', 'rightBottomOnly'], + context: Object.assign(ctx, this.panelAdditionalCtx), isModal: true }); this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ entityName, ...value }) => { @@ -205,9 +210,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.dataSource.loadData(sortedValue); } - private updateErrorText(): void { - if (this.calculatedFieldType === CalculatedFieldType.SIMPLE - && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { + protected updateErrorText(): void { + if (!this.isScript && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; } else if (this.argumentsFormArray.controls.some(control => control.value.refEntityId?.id === NULL_UUID)) { this.errorText = 'calculated-fields.hint.arguments-entity-not-found'; @@ -236,6 +240,14 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces return getEntityDetailsPageURL(id, type); } + protected changeIsScriptMode(): void { + this.argumentsFormArray.updateValueAndValidity(); + } + + protected isEditButtonShowBadge(argument: CalculatedFieldArgumentValue): boolean { + return !(argument.refEntityKey.type === ArgumentType.Rolling && !this.isScript) && argument.refEntityId?.id !== NULL_UUID + } + private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { const value: CalculatedFieldArgumentValue = { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts new file mode 100644 index 0000000000..082001f052 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts @@ -0,0 +1,45 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { + PropagateArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + ], + declarations: [ + CalculatedFieldArgumentPanelComponent, + CalculatedFieldArgumentsTableComponent, + PropagateArgumentsTableComponent + ], + exports: [ + CalculatedFieldArgumentsTableComponent, + PropagateArgumentsTableComponent + ] +}) +export class CalculatedFieldArgumentsTableModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts new file mode 100644 index 0000000000..04d1dbf91b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts @@ -0,0 +1,116 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + OnInit, + Renderer2, + ViewContainerRef, +} from '@angular/core'; +import { FormBuilder, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from '@angular/forms'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityService } from '@core/http/entity.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { ArgumentEntityType, ArgumentType, CalculatedFieldArgumentValue } from '@shared/models/calculated-field.models'; +import { isDefined } from '@core/utils'; +import { NULL_UUID } from '@shared/models/id/has-uuid'; + +@Component({ + selector: 'tb-propagate-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => PropagateArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => PropagateArgumentsTableComponent), + multi: true + } + ], +}) +export class PropagateArgumentsTableComponent extends CalculatedFieldArgumentsTableComponent implements OnInit { + + constructor( + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store + ) { + super(fb, popoverService, viewContainerRef, cd, renderer, entityService, destroyRef, store) + } + + ngOnInit() { + this.updatedValue(); + } + + protected changeIsScriptMode(): void { + this.updatedValue(); + super.changeIsScriptMode(); + } + + private updatedValue() { + if (this.isScript) { + this.argumentNameColumn = 'common.name'; + this.argumentNameColumnCopy = 'calculated-fields.copy-argument-name'; + this.displayColumns = ['name', 'entityType', 'target', 'type', 'key', 'actions']; + this.panelAdditionalCtx = null; + } else { + this.argumentNameColumn = 'calculated-fields.output-key'; + this.argumentNameColumnCopy = 'calculated-fields.copy-output-key'; + this.displayColumns = ['name', 'type', 'key', 'actions']; + this.panelAdditionalCtx = { + argumentEntityTypes: [ArgumentEntityType.Current], + isOutputKey: true + }; + } + } + + protected isEditButtonShowBadge(argument: CalculatedFieldArgumentValue): boolean { + if (!this.isScript && isDefined(argument?.refEntityId)) { + return false; + } + return super.isEditButtonShowBadge(argument); + } + + protected updateErrorText(): void { + if (!this.isScript && this.argumentsFormArray.controls.some(control => isDefined(control.value?.refEntityId))) { + this.errorText = 'calculated-fields.hint.arguments-propagate-argument-entity-type'; + } else if (!this.isScript && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-propagate-arguments-with-rolling'; + } else if (this.argumentsFormArray.controls.some(control => control.value.refEntityId?.id === NULL_UUID)) { + this.errorText = 'calculated-fields.hint.arguments-entity-not-found'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 222c3ffe91..1d9dcc98f1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -67,13 +67,21 @@ } + @case (CalculatedFieldType.PROPAGATION) { + + + } @default { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index cf475111c6..ed0d9dd1c3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -83,6 +83,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent { + if (type !== CalculatedFieldType.SIMPLE && type !== CalculatedFieldType.SCRIPT) { + this.fieldFormGroup.get('configuration').setValue(({} as CalculatedFieldConfiguration), {emitEvent: false}); + } + }); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts index 67c0fe8749..835a3628ff 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component.ts @@ -114,7 +114,7 @@ export class GeofencingConfigurationComponent implements ControlValueAccessor, V } validate(): ValidationErrors | null { - return this.geofencingConfiguration.valid ? null : { geofencingConfigError: false }; + return this.geofencingConfiguration.valid || this.geofencingConfiguration.status === "DISABLED" ? null : { geofencingConfigError: false }; } writeValue(config: CalculatedFieldGeofencingConfiguration): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts index 8fc52d2940..e270dd29cb 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module.ts @@ -28,7 +28,7 @@ import { } from '@home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.component'; import { CalculatedFieldOutputModule -} from '@home/components/calculated-fields/components/output/caclculate-field-output.module'; +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; @NgModule({ imports: [ diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.module.ts similarity index 100% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/output/caclculate-field-output.module.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.html new file mode 100644 index 0000000000..01cc5a542b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.html @@ -0,0 +1,99 @@ + +
+
+
+ {{ 'calculated-fields.propagation-path-related-entities' | translate }} +
+
+ + {{ 'calculated-fields.direction' | translate }} + + @for (direction of Directions; track direction) { + {{ PropagationDirectionTranslations.get(direction) | translate }} + } + + + + +
+
+
+
+
+ {{ 'calculated-fields.data-propagate' | translate }} +
+ + {{ 'calculated-fields.propagate-type.arguments-only' | translate }} + {{ 'calculated-fields.propagate-type.expression-result' | translate }} + +
+ +
+
+
+ {{ 'calculated-fields.expression' | translate }} +
+
+ +
{{ 'api-usage.tbel' | translate }} +
+ + +
+ +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.ts new file mode 100644 index 0000000000..0dfe799bc8 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component.ts @@ -0,0 +1,174 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable, of } from 'rxjs'; +import { + calculatedFieldDefaultScript, + CalculatedFieldOutput, + CalculatedFieldPropagationConfiguration, + CalculatedFieldType, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + PropagationDirectionTranslations, + PropagationWithExpression +} from '@shared/models/calculated-field.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@app/shared/models/rule-node.models'; +import { EntitySearchDirection } from '@shared/models/relation.models'; + +@Component({ + selector: 'tb-propagation-configuration', + templateUrl: './propagation-configuration.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => PropagationConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => PropagationConfigurationComponent), + multi: true + } + ], +}) +export class PropagationConfigurationComponent implements ControlValueAccessor, Validator { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + @Input({required: true}) + testScript: () => Observable; + + propagateConfiguration = this.fb.group({ + arguments: this.fb.control({}), + applyExpressionToResolvedArguments: [false], + direction: [EntitySearchDirection.TO, Validators.required], + relationType: ['Contains', Validators.required], + expression: [calculatedFieldDefaultScript], + output: this.fb.control({ + scope: AttributeScope.SERVER_SCOPE, + type: OutputType.Timeseries, + }), + }); + + readonly ScriptLanguage = ScriptLanguage; + readonly CalculatedFieldType = CalculatedFieldType; + readonly OutputType = OutputType; + readonly Directions = Object.values(EntitySearchDirection) as Array; + readonly PropagationDirectionTranslations = PropagationDirectionTranslations; + + functionArgs$ = this.propagateConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.propagateConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj ?? {})) + ); + + argumentsHighlightRules$ = this.propagateConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + private propagateChange: (config: CalculatedFieldPropagationConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder) { + this.propagateConfiguration.get('applyExpressionToResolvedArguments').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(() => { + this.updatedFormWithScript(); + }) + + this.propagateConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value: CalculatedFieldPropagationConfiguration) => { + this.updatedModel(value); + }) + } + + validate(): ValidationErrors | null { + return this.propagateConfiguration.valid || this.propagateConfiguration.status === "DISABLED" ? null : {invalidPropagateConfig: false}; + } + + writeValue(value: PropagationWithExpression): void { + value.expression = value.expression ?? calculatedFieldDefaultScript; + this.propagateConfiguration.patchValue(value, {emitEvent: false}); + this.updatedFormWithScript(); + setTimeout(() => { + this.propagateConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); + } + + registerOnChange(fn: (config: CalculatedFieldPropagationConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.propagateConfiguration.disable({emitEvent: false}); + } else { + this.propagateConfiguration.enable({emitEvent: false}); + this.updatedFormWithScript(); + } + } + + onTestScript() { + this.testScript().subscribe((expression) => { + this.propagateConfiguration.get('expression').setValue(expression); + this.propagateConfiguration.get('expression').markAsDirty(); + }) + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(['Contains', 'Manages']).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private updatedModel(value: CalculatedFieldPropagationConfiguration): void { + value.type = CalculatedFieldType.PROPAGATION; + this.propagateChange(value); + } + + private updatedFormWithScript() { + if (this.propagateConfiguration.get('applyExpressionToResolvedArguments').value) { + this.propagateConfiguration.get('expression').enable({emitEvent: false}); + } else { + this.propagateConfiguration.get('expression').disable({emitEvent: false}); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module.ts new file mode 100644 index 0000000000..83dc4badf9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; +import { + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; +import { + PropagationConfigurationComponent +} from '@home/components/calculated-fields/components/propagation-configuration/propagation-configuration.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, + ], + declarations: [ + PropagationConfigurationComponent, + ], + exports: [ + PropagationConfigurationComponent, + ] +}) +export class PropagationConfigurationModule { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html index a4c37bcdee..a44e4362af 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.html @@ -22,7 +22,7 @@ [entityId]="entityId" [tenantId]="tenantId" [entityName]="entityName" - [calculatedFieldType]="(isScript ? CalculatedFieldType.SCRIPT : CalculatedFieldType.SIMPLE)" /> + [isScript]="isScript" />
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts index 42137cac1c..89b720bd7e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.component.ts @@ -66,17 +66,17 @@ export class SimpleConfigurationComponent implements ControlValueAccessor, Valid @Input() isScript: boolean; - @Input() + @Input({required: true}) entityId: EntityId; - @Input() + @Input({required: true}) tenantId: string; - @Input() + @Input({required: true}) entityName: string; - @Input() - testScript$: Observable; + @Input({required: true}) + testScript: () => Observable; simpleConfiguration = this.fb.group({ arguments: this.fb.control({}), @@ -92,7 +92,6 @@ export class SimpleConfigurationComponent implements ControlValueAccessor, Valid }); readonly ScriptLanguage = ScriptLanguage; - readonly CalculatedFieldType = CalculatedFieldType; readonly OutputType = OutputType; functionArgs$ = this.simpleConfiguration.get('arguments').valueChanges.pipe( @@ -141,19 +140,21 @@ export class SimpleConfigurationComponent implements ControlValueAccessor, Valid } validate(): ValidationErrors | null { - return this.simpleConfiguration.valid ? null : {invalidSimpleConfig: false}; + return this.simpleConfiguration.valid || this.simpleConfiguration.status === "DISABLED" ? null : {invalidSimpleConfig: false}; } writeValue(value: SimpeConfiguration): void { const formValue: any = deepClone(value); if (this.isScript) { - formValue.expressionSCRIPT = formValue.expression; + formValue.expressionSCRIPT = formValue.expression ?? calculatedFieldDefaultScript; } else { formValue.expressionSIMPLE = formValue.expression; } this.simpleConfiguration.patchValue(formValue, {emitEvent: false}); - this.simpleConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); this.updatedFormWithScript(); + setTimeout(() => { + this.simpleConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); } registerOnChange(fn: (config: SimpeConfiguration) => void): void { @@ -173,7 +174,7 @@ export class SimpleConfigurationComponent implements ControlValueAccessor, Valid } onTestScript() { - this.testScript$?.subscribe((expression) => { + this.testScript().subscribe((expression) => { this.simpleConfiguration.get('expressionSCRIPT').setValue(expression); this.simpleConfiguration.get('expressionSCRIPT').markAsDirty(); }) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts index aee32a0916..2e5e14426e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/simple-configuration/simple-configuration.module.ts @@ -20,26 +20,22 @@ import { SharedModule } from '@shared/shared.module'; import { SimpleConfigurationComponent } from '@home/components/calculated-fields/components/simple-configuration/simple-configuration.component'; -import { - CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-argument-panel.component'; import { CalculatedFieldOutputModule -} from '@home/components/calculated-fields/components/output/caclculate-field-output.module'; +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; import { - CalculatedFieldArgumentsTableComponent -} from '@home/components/calculated-fields/components/simple-configuration/calculated-field-arguments-table.component'; + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; @NgModule({ imports: [ CommonModule, SharedModule, CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, ], declarations: [ SimpleConfigurationComponent, - CalculatedFieldArgumentPanelComponent, - CalculatedFieldArgumentsTableComponent ], exports: [ SimpleConfigurationComponent diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8a2c34d036..8294a776df 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -50,10 +50,16 @@ export interface CalculatedFieldGeofencing extends BaseCalculatedField { configuration: CalculatedFieldGeofencingConfiguration; } +export interface CalculatedFieldPropagation extends BaseCalculatedField { + type: CalculatedFieldType.PROPAGATION; + configuration: CalculatedFieldPropagationConfiguration; +} + export type CalculatedField = | CalculatedFieldSimple | CalculatedFieldScript - | CalculatedFieldGeofencing; + | CalculatedFieldGeofencing + | CalculatedFieldPropagation; export enum CalculatedFieldType { SIMPLE = 'SIMPLE', @@ -74,30 +80,52 @@ export const CalculatedFieldTypeTranslations = new Map; + expression: string; + arguments: Record; output: CalculatedFieldSimpleOutput; } export interface CalculatedFieldScriptConfiguration { type: CalculatedFieldType.SCRIPT; - expression?: string; - arguments?: Record; + expression: string; + arguments: Record; output: CalculatedFieldOutput; } export interface CalculatedFieldGeofencingConfiguration { type: CalculatedFieldType.GEOFENCING; - zoneGroups?: Record; - scheduledUpdateEnabled?: boolean; + zoneGroups: Record; + scheduledUpdateEnabled: boolean; scheduledUpdateInterval?: number; output: CalculatedFieldOutput; } +interface BasePropagationConfiguration { + type: CalculatedFieldType.PROPAGATION; + direction: EntitySearchDirection; + relationType: string; + arguments: Record; + output: CalculatedFieldOutput; +} + +export interface PropagationWithNoExpression extends BasePropagationConfiguration { + applyExpressionToResolvedArguments: false; +} + +export interface PropagationWithExpression extends BasePropagationConfiguration { + applyExpressionToResolvedArguments: true; + expression: string; +} + +export type CalculatedFieldPropagationConfiguration = + | PropagationWithNoExpression + | PropagationWithExpression; + export interface CalculatedFieldOutput { type: OutputType; scope?: AttributeScope; @@ -156,6 +184,13 @@ export const GeofencingDirectionLevelTranslations = new Map( + [ + [EntitySearchDirection.FROM, 'calculated-fields.direction-down-child'], + [EntitySearchDirection.TO, 'calculated-fields.direction-up-parent'], + ] +) + export enum ArgumentType { Attribute = 'ATTRIBUTE', LatestTelemetry = 'TS_LATEST', diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 24266ad192..7f728c0b63 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1064,6 +1064,7 @@ "datasource": "Datasource", "add-argument": "Add argument", "test-script-function": "Test script function", + "test-expression-function": "Test expression function", "no-arguments": "No arguments configured", "argument-settings": "Argument settings", "argument-current": "Current entity", @@ -1139,14 +1140,26 @@ "level": "Level", "direction-level": "Direction", "direction-up": "Up", + "direction-up-parent": "Up to parent", "direction-down": "Down", + "direction-down-child": "Down to child", "add-level": "Add level", "delete-level": "Delete level", "no-level": "No level configured", "levels-required": "At least one level must be configured.", "max-allowed-levels-error": "Relation level exceeds the maximum allowed.", + "propagation-path-related-entities": "Propagation path to related entities", + "propagate-type": { + "arguments-only": "Arguments only", + "expression-result": "Expression result" + }, + "data-propagate": "Data to propagate", + "output-key": "Output key", + "copy-output-key": "Copy output key", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", + "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", + "arguments-propagate-argument-entity-type": "Entity type is incompatible with 'Arguments only' propagation.", "arguments-empty": "Arguments should not be empty.", "expression-required": "Expression is required.", "expression-invalid": "Expression is invalid", @@ -1156,6 +1169,12 @@ "argument-name-duplicate": "Argument with such name already exists.", "argument-name-max-length": "Argument name should be less than 256 characters.", "argument-name-forbidden": "Argument name is reserved and cannot be used.", + "output-key-required": "Output key is required.", + "output-key-pattern": "Output key is invalid.", + "output-key-duplicate": "Key with such name already exists.", + "output-key-max-length": "Output key should be less than 256 characters.", + "output-key-forbidden": "Output key is reserved and cannot be used.", + "entity-type-required": "Entity type is required", "name-required": "Mame is required.", "name-pattern": "Name is invalid.", "name-duplicate": "Name with such name already exists.", @@ -1181,7 +1200,9 @@ "max-geofencing-zone": "Maximum number of geofencing zones reached.", "zone-group-refresh-interval": "Defines how often zone groups configured via related entities are refreshed.", "zone-group-refresh-interval-required": "Zone groups refresh interval is required.", - "zone-group-refresh-interval-min": "Zone group refresh interval should be at least {{ min }} second." + "zone-group-refresh-interval-min": "Zone group refresh interval should be at least {{ min }} second.", + "propagation-path-related-entities": "Defines a direct, single-level path to a related entity based on the selected direction and relation type.", + "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data." } }, "ai-models": { From e6479d5856e4c8860e674d2d309a502c978aa25b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 12:38:37 +0300 Subject: [PATCH 048/122] removed relation by profile processing --- .../processing/AbstractConsumerService.java | 16 +-- .../server/dao/relation/RelationService.java | 15 --- .../configuration/aggregation/AggSource.java | 30 ----- .../ProfileEntityRelationPathQuery.java | 21 ---- .../dao/relation/BaseRelationService.java | 100 ---------------- .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 7 -- .../dao/sql/relation/JpaRelationDao.java | 108 ------------------ .../dao/sql/relation/RelationRepository.java | 36 ------ 9 files changed, 5 insertions(+), 335 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 36a35d9e44..6e162256a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -188,13 +188,9 @@ public abstract class AbstractConsumerService> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); - - List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - - ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - - List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); - - void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); - // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java deleted file mode 100644 index 84f6f92eb3..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.relation.RelationPathLevel; - -import java.util.List; - -@Data -public class AggSource { - - private RelationPathLevel relation; - private List entityProfiles; - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java deleted file mode 100644 index 32b338ff6f..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.relation; - -import org.thingsboard.server.common.data.id.EntityId; - -public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 2d584ebebe..18d1806fe4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,7 +45,6 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,105 +514,6 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } - @Override - public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - RelationPathLevel relationPathLevel = relationPathQuery.level(); - return switch (relationPathLevel.direction()) { - case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - }; - } - - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); -// RelationPathLevel relationPathLevel = relationPathQuery.level(); -// return switch (relationPathLevel.direction()) { -// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// }; - } - - @Override - public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); - validate(from); - validateType(relationType); - if (targetProfileId == null) { - return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); - } - - @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); - validate(to); - validateType(relationType); - if (targetProfileId == null) { - return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); - } - - @Override - public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { - -// List keys = new ArrayList<>(5); -// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); -// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); -// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); -// cache.evict(keys); -// log.debug("Processed evict event: {}", event); - - List keys = new ArrayList<>(2); - keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); - keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); - cache.evict(keys); - log.debug("Processed evict relations by keys: {}", keys); - } - - private void validate(ProfileEntityRelationPathQuery relationPathQuery) { - validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); - relationPathQuery.level().validate(); - } - private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index 344af0a6f3..d6f0525c9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,14 +40,9 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; - private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null, null); - } - - public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { - this(from, to, type, typeGroup, direction, null); + this(from, to, type, typeGroup, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 6318f1fc9d..ad53164ad7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -37,12 +36,8 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); - List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); @@ -79,6 +74,4 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index afecca26c1..b2871313ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -42,12 +41,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -107,11 +103,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -121,17 +112,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList( - relationRepository.findByToAndProfile( - to.getId(), - to.getEntityType().name(), - typeGroup.name(), - relationType, - profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( @@ -412,92 +392,4 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { - String sql = buildProfileEntityRelationPathSql(query); - Object[] params = buildProfileEntityRelationPathParams(query); - - log.trace("[{}] profile entity relation path query: {}", tenantId, sql); - - return jdbcTemplate.queryForList(sql, params).stream() - .map(row -> { - var entityRelation = new EntityRelation(); - var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); - var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); - var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); - var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); - var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); - var type = (String) row.get(RELATION_TYPE_PROPERTY); - var version = (Long) row.get(VERSION_COLUMN); - - entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); - entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); - entityRelation.setType(type); - entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); - entityRelation.setVersion(version); - return entityRelation; - }) - .collect(Collectors.toList()); - } - - private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { - final List params = new ArrayList<>(); - - params.add(query.rootEntityId().getId()); - params.add(query.rootEntityId().getEntityType().name()); - - params.add(query.level().relationType()); - - if (query.targetEntityProfileId() != null) { - params.add(query.targetEntityProfileId().getId()); - params.add(query.targetEntityProfileId().getId()); - } - - return params.toArray(); - } - - private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { - EntitySearchDirection direction = query.level().direction(); - - StringBuilder sb = new StringBuilder(); - - sb.append("\n") - .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") - .append(" r.relation_type_group, r.relation_type, r.version\n") - .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); - - sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); - } else { - sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); - } - - sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); - } else { - sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); - } - - if (EntitySearchDirection.FROM == direction) { - sb.append("WHERE r.from_id = ?").append("\n") - .append("AND r.from_type = ?").append("\n"); - } else { - sb.append("WHERE r.to_id = ?").append("\n") - .append("AND r.to_type = ?").append("\n"); - } - - sb.append("AND r.relation_type = ?").append("\n") - .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); - - if (query.targetEntityProfileId() != null) { - sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); - } - - sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); - - return sb.toString(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 0ebd5b6ceb..4b879f9d95 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; -import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -97,39 +96,4 @@ public interface RelationRepository @Param("toType") String toType, @Param("batchSize") int batchSize); - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' - LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' - WHERE r.from_id = :fromId - AND r.from_type = :fromType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByFromAndProfile(@Param("fromId") UUID fromId, - @Param("fromType") String fromType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); - - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' - LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' - WHERE r.to_id = :toId - AND r.to_type = :toType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByToAndProfile(@Param("toId") UUID toId, - @Param("toType") String toType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); } From 6da6f846521dfd2e4daa7fe57aa5ebfaee9a9f8d Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 16 Oct 2025 13:58:19 +0300 Subject: [PATCH 049/122] UI: refactor cf module --- .../calculated-fields/calculated-field.module.ts | 11 ++--------- .../modules/home/components/home-components.module.ts | 9 +++++++++ .../home/pages/asset-profile/asset-profile.module.ts | 2 -- .../src/app/modules/home/pages/asset/asset.module.ts | 2 -- .../pages/device-profile/device-profile.module.ts | 2 -- .../app/modules/home/pages/device/device.module.ts | 2 -- 6 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index 6cb6a907b7..e0db7a6eef 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -17,13 +17,9 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { SharedModule } from '@shared/shared.module'; -import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; -import { - CalculatedFieldDebugDialogComponent -} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; import { CalculatedFieldScriptTestDialogComponent } from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; @@ -33,7 +29,6 @@ import { import { EntityDebugSettingsButtonComponent } from '@home/components/entity/debug/entity-debug-settings-button.component'; -import { HomeComponentsModule } from '@home/components/home-components.module'; import { GeofencingConfigurationModule } from '@home/components/calculated-fields/components/geofencing-configuration/geofencing-configuration.module'; @@ -46,9 +41,7 @@ import { @NgModule({ declarations: [ - CalculatedFieldsTableComponent, CalculatedFieldDialogComponent, - CalculatedFieldDebugDialogComponent, CalculatedFieldScriptTestDialogComponent, CalculatedFieldTestArgumentsComponent, ], @@ -57,12 +50,12 @@ import { SharedModule, GeofencingConfigurationModule, EntityDebugSettingsButtonComponent, - HomeComponentsModule, SimpleConfigurationModule, PropagationConfigurationModule, ], exports: [ - CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent, ] }) export class CalculatedFieldsModule {} diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 2c9c08aeb3..e15d466b7a 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -190,6 +190,11 @@ import { CheckConnectivityDialogComponent } from '@home/components/ai-model/chec import { AIModelDialogComponent } from '@home/components/ai-model/ai-model-dialog.component'; import { ResourcesDialogComponent } from "@home/components/resources/resources-dialog.component"; import { ResourcesLibraryComponent } from "@home/components/resources/resources-library.component"; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: @@ -202,6 +207,8 @@ import { ResourcesLibraryComponent } from "@home/components/resources/resources- EntityDetailsPageComponent, AuditLogTableComponent, AuditLogDetailsDialogComponent, + CalculatedFieldsTableComponent, + CalculatedFieldDebugDialogComponent, EventContentDialogComponent, EventTableHeaderComponent, EventTableComponent, @@ -343,6 +350,7 @@ import { ResourcesLibraryComponent } from "@home/components/resources/resources- CommonModule, SharedModule, SharedHomeComponentsModule, + CalculatedFieldsModule, WidgetConfigComponentsModule, BasicWidgetConfigModule, Lwm2mProfileComponentsModule, @@ -360,6 +368,7 @@ import { ResourcesLibraryComponent } from "@home/components/resources/resources- EntityDetailsPanelComponent, EntityDetailsPageComponent, AuditLogTableComponent, + CalculatedFieldsTableComponent, EventTableComponent, EdgeDownlinkTableHeaderComponent, EdgeDownlinkTableComponent, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts index c174a2b97b..fb10712d57 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile.module.ts @@ -20,7 +20,6 @@ import { SharedModule } from '@shared/shared.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { AssetProfileTabsComponent } from './asset-profile-tabs.component'; import { AssetProfileRoutingModule } from './asset-profile-routing.module'; -import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -30,7 +29,6 @@ import { CalculatedFieldsModule } from '@home/components/calculated-fields/calcu CommonModule, SharedModule, HomeComponentsModule, - CalculatedFieldsModule, AssetProfileRoutingModule, ] }) diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts b/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts index 44fa22520f..17c8317fe2 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.module.ts @@ -23,7 +23,6 @@ import { AssetTableHeaderComponent } from './asset-table-header.component'; import { AssetRoutingModule } from './asset-routing.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { AssetTabsComponent } from '@home/pages/asset/asset-tabs.component'; -import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -36,7 +35,6 @@ import { CalculatedFieldsModule } from '@home/components/calculated-fields/calcu SharedModule, HomeComponentsModule, HomeDialogsModule, - CalculatedFieldsModule, AssetRoutingModule, ] }) diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts index 12b68f4ab4..76d15d00f1 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile.module.ts @@ -20,7 +20,6 @@ import { SharedModule } from '@shared/shared.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { DeviceProfileTabsComponent } from './device-profile-tabs.component'; import { DeviceProfileRoutingModule } from './device-profile-routing.module'; -import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -30,7 +29,6 @@ import { CalculatedFieldsModule } from '@home/components/calculated-fields/calcu CommonModule, SharedModule, HomeComponentsModule, - CalculatedFieldsModule, DeviceProfileRoutingModule ] }) diff --git a/ui-ngx/src/app/modules/home/pages/device/device.module.ts b/ui-ngx/src/app/modules/home/pages/device/device.module.ts index 8681ff7fe7..4c74da0f89 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device.module.ts @@ -36,7 +36,6 @@ import { SnmpDeviceTransportConfigurationComponent } from './data/snmp-device-tr import { DeviceCredentialsModule } from '@home/components/device/device-credentials.module'; import { DeviceProfileCommonModule } from '@home/components/profile/device/common/device-profile-common.module'; import { DeviceCheckConnectivityDialogComponent } from './device-check-connectivity-dialog.component'; -import { CalculatedFieldsModule } from '@home/components/calculated-fields/calculated-field.module'; @NgModule({ declarations: [ @@ -62,7 +61,6 @@ import { CalculatedFieldsModule } from '@home/components/calculated-fields/calcu HomeDialogsModule, DeviceCredentialsModule, DeviceProfileCommonModule, - CalculatedFieldsModule, DeviceRoutingModule ] }) From 25ac2f2b487c48723b0f7d4c6ae8837b6b6de14e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 16:32:35 +0300 Subject: [PATCH 050/122] added more tests and changed state proto --- .../cf/ctx/state/CalculatedFieldCtx.java | 15 +- ...ValuesAggregationCalculatedFieldState.java | 5 +- .../service/cf/ctx/state/aggregation/agg.json | 65 ------ .../server/utils/CalculatedFieldUtils.java | 4 + ...tValuesAggregationCalculatedFieldTest.java | 188 +++++++++++++++++- common/proto/src/main/proto/queue.proto | 1 + 6 files changed, 196 insertions(+), 82 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 3a9d75b3d4..f69d611b64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,8 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() - && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; @@ -596,7 +595,7 @@ public class CalculatedFieldCtx { var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || - !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { + !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { return true; } } @@ -611,7 +610,7 @@ public class CalculatedFieldCtx { private boolean hasGeofencingZoneGroupConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { + && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { return !thisConfig.getZoneGroups().equals(otherConfig.getZoneGroups()); } return false; @@ -663,10 +662,10 @@ public class CalculatedFieldCtx { @Override public String toString() { return "CalculatedFieldCtx{" + - "cfId=" + cfId + - ", cfType=" + cfType + - ", entityId=" + entityId + - '}'; + "cfId=" + cfId + + ", cfType=" + cfType + + ", entityId=" + entityId + + '}'; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 1df27b4cbd..6ae023ed1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -48,6 +48,7 @@ import java.util.Map.Entry; @Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + @Setter private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; @@ -74,6 +75,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; + inputs.clear(); } @Override @@ -104,7 +106,8 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - if (!shouldRecalculate()) { + boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); + if (!shouldRecalculate() && !shouldRecalculate) { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json deleted file mode 100644 index b39092ca74..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET_PROFILE", - "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "source": { - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "entityProfiles": [ - { - "entityType": "DEVICE_PROFILE", - "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" - } - ] - }, - "inputs": { - "oc": { - "key": "occupied", - "type": "TS_LATEST" - } - }, - "deduplicationIntervalMillis": 10000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 19ad4bfb7c..00f4cbde0f 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -96,6 +96,9 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + } state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() @@ -242,6 +245,7 @@ public class CalculatedFieldUtils { arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); + aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index a4b43e362b..8c2c2ca68c 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.cf; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +64,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; +@Slf4j @DaoSqlTest public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { @@ -341,13 +345,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - postTelemetry(device4.getId(), "{\"occupied\":true}"); - postTelemetry(device3.getId(), "{\"occupied\":true}"); + long currentTime = System.currentTimeMillis(); + long firstTs = currentTime - 10; + long secondTs = currentTime - 10; + long thirdTs = currentTime - 5; + postTelemetry(device3.getId(), "{\"ts\": " + firstTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device4.getId(), "{\"ts\": " + secondTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device3.getId(), "{\"ts\": " + thirdTs + ", \"values\": {\"occupied\":true}}"); createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -357,15 +365,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll )); }); - doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( - "freeSpaces", "2", - "occupiedSpaces", "0", + "freeSpaces", "1", + "occupiedSpaces", "1", "totalSpaces", "2" )); }); @@ -411,6 +419,140 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testUpdateRelationPath_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + Device device3 = createDevice("Device 3", "1234567890333"); + createEntityRelation(asset.getId(), device3.getId(), "Has"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); + saveCalculatedField(cf); + + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "1", + "totalSpaces", "1" + )); + }); + } + + @Test + public void testUpdateArguments_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); + postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + configuration.setArguments(Map.of("oc", argument)); + saveCalculatedField(cf); + + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateMetrics_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + AggMetric aggMetric = new AggMetric(); + aggMetric.setInput(new AggKeyInput("temp")); + aggMetric.setFilter("return temp < 100;"); + aggMetric.setFunction(AggFunction.MAX); + configuration.setMetrics(Map.of("maxTemperature", aggMetric)); + saveCalculatedField(cf); + + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); + postTelemetry(device2.getId(), "{\"temperature\":25.8}"); + + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); + }); + } + + @Test + public void testUpdateOutput_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + configuration.setOutput(output); + saveCalculatedField(cf); + + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); + assertThat(avgTemperature).isNotNull(); + assertThat(avgTemperature.get(0)).isNotNull(); + assertThat(avgTemperature.get(0).get("value").asText()).isEqualTo("24.2"); + }); + } + + @Test + public void testUpdateDeduplicationInterval_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + saveCalculatedField(cf); + + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); + + await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); + }); + } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -425,6 +567,32 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } + private CalculatedField createAvgTemperatureCF(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("20"); + arguments.put("temp", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric avgMetric = new AggMetric(); + avgMetric.setFunction(AggFunction.AVG); + avgMetric.setFilter("return temp >= 20;"); + avgMetric.setInput(new AggKeyInput("temp")); + aggMetrics.put("avgTemperature", avgMetric); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Average temperature", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); @@ -513,4 +681,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 764f4c3ec9..1e01916e55 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -928,6 +928,7 @@ message CalculatedFieldStateProto { repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; repeated AggSingleArgumentEntryProto aggArguments = 7; + int64 lastArgsUpdateTs = 8; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 57372035cc255e957c2e3748e6f27df20ef78f36 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 16 Oct 2025 16:34:21 +0300 Subject: [PATCH 051/122] Fixed typo --- .../thingsboard/server/dao/relation/BaseRelationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index eecb31713e..4e4e86a6d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -511,7 +511,7 @@ public class BaseRelationService implements RelationService { validateId(tenantId, id -> "Invalid tenant id: " + id); validate(relationPathQuery); int limit = (int) apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxRelatedEntitiesToReturnPerCfArgument); - validatePositiveNumber(limit, "Invalid entities limit: " + limit); + validatePositiveNumber(limit, "Max related entities limit for relation path query must be positive!"); if (relationPathQuery.levels().size() == 1) { RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); var relationsFuture = switch (relationPathLevel.direction()) { From 58a4600342d65281eaac29adc3adf6f003e82314 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 16 Oct 2025 16:36:26 +0300 Subject: [PATCH 052/122] fix typo in upgrade script --- application/src/main/data/upgrade/basic/schema_update.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 2be18eaf35..0862a33926 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -40,7 +40,7 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument' THEN NULL ELSE to_jsonb(100) - END, + END ) ), false From 6b7faf94a9257e11e4068c61b173046aa2c50bcc Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 16 Oct 2025 19:20:38 +0300 Subject: [PATCH 053/122] UI: Add new property in tenant profile --- ...eofencing-zone-groups-panel.component.html | 2 +- ...enant-profile-configuration.component.html | 216 ++++++++++-------- ...-tenant-profile-configuration.component.ts | 216 ++++++++---------- ui-ngx/src/app/shared/models/tenant.model.ts | 11 + .../assets/locale/locale.constant-en_US.json | 4 + 5 files changed, 232 insertions(+), 217 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.html index a460deaf4e..1e8483f1cc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.html @@ -176,7 +176,7 @@ - @if (entityFilter.singleEntity.id) { + @if (entityFilter.singleEntity?.id) {
{{ 'calculated-fields.perimeter-attribute-key' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 1540a3c87f..d53cacbd83 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{ 'tenant-profile.entities' | translate }} tenant-profile.unlimited @@ -26,10 +26,10 @@ - + {{ 'tenant-profile.maximum-devices-required' | translate}} - + {{ 'tenant-profile.maximum-devices-range' | translate}} @@ -39,10 +39,10 @@ - + {{ 'tenant-profile.maximum-dashboards-required' | translate}} - + {{ 'tenant-profile.maximum-dashboards-range' | translate}} @@ -54,10 +54,10 @@ - + {{ 'tenant-profile.maximum-assets-required' | translate}} - + {{ 'tenant-profile.maximum-assets-range' | translate}} @@ -67,10 +67,10 @@ - + {{ 'tenant-profile.maximum-users-required' | translate}} - + {{ 'tenant-profile.maximum-users-range' | translate}} @@ -89,10 +89,10 @@ - + {{ 'tenant-profile.maximum-customers-required' | translate}} - + {{ 'tenant-profile.maximum-customers-range' | translate}} @@ -102,10 +102,10 @@ - + {{ 'tenant-profile.maximum-rule-chains-required' | translate}} - + {{ 'tenant-profile.maximum-rule-chains-range' | translate}} @@ -117,10 +117,10 @@ - + {{ 'tenant-profile.maximum-edges-required' | translate }} - + {{ 'tenant-profile.maximum-edges-range' | translate }} @@ -141,10 +141,10 @@ - + {{ 'tenant-profile.max-r-e-executions-required' | translate}} - + {{ 'tenant-profile.max-r-e-executions-range' | translate}} @@ -154,10 +154,10 @@ - + {{ 'tenant-profile.max-transport-messages-required' | translate}} - + {{ 'tenant-profile.max-transport-messages-range' | translate}} @@ -176,10 +176,10 @@ - + {{ 'tenant-profile.max-j-s-executions-required' | translate}} - + {{ 'tenant-profile.max-j-s-executions-range' | translate}} @@ -189,10 +189,10 @@ - + {{ 'tenant-profile.max-tbel-executions-required' | translate}} - + {{ 'tenant-profile.max-tbel-executions-range' | translate}} @@ -204,10 +204,10 @@ - + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} - + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} @@ -217,10 +217,10 @@ - + {{ 'tenant-profile.max-transport-data-points-required' | translate}} - + {{ 'tenant-profile.max-transport-data-points-range' | translate}} @@ -239,10 +239,10 @@ - + {{ 'tenant-profile.max-calculated-fields-required' | translate}} - + {{ 'tenant-profile.max-calculated-fields-range' | translate}} @@ -252,10 +252,10 @@ - + {{ 'tenant-profile.max-data-points-per-rolling-arg-required' | translate}} - + {{ 'tenant-profile.max-data-points-per-rolling-arg-range' | translate}} @@ -267,42 +267,14 @@ - + {{ 'tenant-profile.max-arguments-per-cf-required' | translate}} - + {{ 'tenant-profile.max-arguments-per-cf-range' | translate}} - - tenant-profile.max-related-level-per-argument - - - {{ 'tenant-profile.max-related-level-per-argument-required' | translate}} - - - {{ 'tenant-profile.max-related-level-per-argument-range' | translate}} - - - -
-
- - tenant-profile.min-allowed-scheduled-update-interval - - - {{ 'tenant-profile.min-allowed-scheduled-update-interval-required' | translate}} - - - {{ 'tenant-profile.min-allowed-scheduled-update-interval-range' | translate}} - - -
@@ -318,10 +290,10 @@ - + {{ 'tenant-profile.max-state-size-required' | translate}} - + {{ 'tenant-profile.max-state-size-range' | translate}} @@ -331,15 +303,59 @@ - + {{ 'tenant-profile.max-value-argument-size-required' | translate}} - + {{ 'tenant-profile.max-value-argument-size-range' | translate}}
+
+ + tenant-profile.max-related-level-per-argument + + + {{ 'tenant-profile.max-related-level-per-argument-required' | translate}} + + + {{ 'tenant-profile.max-related-level-per-argument-range' | translate}} + + + + + tenant-profile.min-allowed-scheduled-update-interval + + + {{ 'tenant-profile.min-allowed-scheduled-update-interval-required' | translate}} + + + {{ 'tenant-profile.min-allowed-scheduled-update-interval-range' | translate}} + + + +
+
+ + tenant-profile.relation-search-entity-limit + + + {{ 'tenant-profile.relation-search-entity-limit-required' | translate}} + + + {{ 'tenant-profile.relation-search-entity-limit-range' | translate}} + + tenant-profile.relation-search-entity-limit-hint + +
+
@@ -354,10 +370,10 @@ - + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} - + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} @@ -367,10 +383,10 @@ - + {{ 'tenant-profile.alarms-ttl-days-required' | translate}} - + {{ 'tenant-profile.alarms-ttl-days-days-range' | translate}} @@ -382,10 +398,10 @@ - + {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} - + {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} @@ -395,10 +411,10 @@ - + {{ 'tenant-profile.rpc-ttl-days-required' | translate}} - + {{ 'tenant-profile.rpc-ttl-days-days-range' | translate}} @@ -410,10 +426,10 @@ - + {{ 'tenant-profile.queue-stats-ttl-days-required' | translate}} - + {{ 'tenant-profile.queue-stats-ttl-days-range' | translate}} @@ -423,10 +439,10 @@ - + {{ 'tenant-profile.rule-engine-exceptions-ttl-days-required' | translate}} - + {{ 'tenant-profile.rule-engine-exceptions-ttl-days-range' | translate}} @@ -441,16 +457,16 @@ {{ 'tenant-profile.sms-enabled' | translate }} - tenant-profile.max-sms - + {{ 'tenant-profile.max-sms-required' | translate}} - + {{ 'tenant-profile.max-sms-range' | translate}} @@ -460,10 +476,10 @@ - + {{ 'tenant-profile.max-emails-required' | translate}} - + {{ 'tenant-profile.max-emails-range' | translate}} @@ -473,10 +489,10 @@ - + {{ 'tenant-profile.max-created-alarms-required' | translate}} - + {{ 'tenant-profile.max-created-alarms-range' | translate}} @@ -494,7 +510,7 @@ - + {{ 'tenant-profile.maximum-debug-duration-min-range' | translate }} @@ -513,10 +529,10 @@ - + {{ 'tenant-profile.maximum-resources-sum-data-size-required' | translate}} - + {{ 'tenant-profile.maximum-resources-sum-data-size-range' | translate}} @@ -526,10 +542,10 @@ - + {{ 'tenant-profile.maximum-ota-package-sum-data-size-required' | translate}} - + {{ 'tenant-profile.maximum-ota-package-sum-data-size-range' | translate}} @@ -541,10 +557,10 @@ - + {{ 'tenant-profile.maximum-resource-size-required' | translate}} - + {{ 'tenant-profile.maximum-resource-size-range' | translate}} @@ -561,14 +577,14 @@ tenant-profile.ws-limit-max-sessions-per-tenant - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-tenant - + {{ 'tenant-profile.too-small-value-zero' | translate}} @@ -577,14 +593,14 @@ tenant-profile.ws-limit-max-sessions-per-customer - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-customer - + {{ 'tenant-profile.too-small-value-zero' | translate}} @@ -600,14 +616,14 @@ tenant-profile.ws-limit-max-sessions-per-public-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-public-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} @@ -616,14 +632,14 @@ tenant-profile.ws-limit-max-sessions-per-regular-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} tenant-profile.ws-limit-max-subscriptions-per-regular-user - + {{ 'tenant-profile.too-small-value-zero' | translate}} @@ -632,7 +648,7 @@ tenant-profile.ws-limit-queue-per-session - + {{ 'tenant-profile.too-small-value-one' | translate}} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 0dd1453648..9595def95e 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -14,16 +14,13 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; -import { ControlValueAccessor, UntypedFormBuilder, UntypedFormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; -import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { DefaultTenantProfileConfiguration, TenantProfileConfiguration } from '@shared/models/tenant.model'; +import { Component, forwardRef, Input } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { DefaultTenantProfileConfiguration, FormControlsFrom } from '@shared/models/tenant.model'; import { isDefinedAndNotNull } from '@core/utils'; import { RateLimitsType } from './rate-limits/rate-limits.models'; -import { takeUntil } from 'rxjs/operators'; -import { Subject } from 'rxjs'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { coerceBoolean } from '@shared/decorators/coercion'; @Component({ selector: 'tb-default-tenant-profile-configuration', @@ -35,112 +32,107 @@ import { Subject } from 'rxjs'; multi: true }] }) -export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor, OnInit, OnDestroy { +export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor { - defaultTenantProfileConfigurationFormGroup: UntypedFormGroup; + tenantProfileConfigurationForm: FormGroup>; - private requiredValue: boolean; - private destroy$ = new Subject(); - get required(): boolean { - return this.requiredValue; - } @Input() - set required(value: boolean) { - this.requiredValue = coerceBooleanProperty(value); - } + @coerceBoolean() + required: boolean; @Input() + @coerceBoolean() disabled: boolean; rateLimitsType = RateLimitsType; - private propagateChange = (v: any) => { }; - - constructor(private store: Store, - private fb: UntypedFormBuilder) { - this.defaultTenantProfileConfigurationFormGroup = this.fb.group({ - maxDevices: [null, [Validators.required, Validators.min(0)]], - maxAssets: [null, [Validators.required, Validators.min(0)]], - maxCustomers: [null, [Validators.required, Validators.min(0)]], - maxUsers: [null, [Validators.required, Validators.min(0)]], - maxDashboards: [null, [Validators.required, Validators.min(0)]], - maxRuleChains: [null, [Validators.required, Validators.min(0)]], - maxEdges: [null, [Validators.required, Validators.min(0)]], - maxResourcesInBytes: [null, [Validators.required, Validators.min(0)]], - maxOtaPackagesInBytes: [null, [Validators.required, Validators.min(0)]], - maxResourceSize: [null, [Validators.required, Validators.min(0)]], - transportTenantMsgRateLimit: [null, []], - transportTenantTelemetryMsgRateLimit: [null, []], - transportTenantTelemetryDataPointsRateLimit: [null, []], - transportDeviceMsgRateLimit: [null, []], - transportDeviceTelemetryMsgRateLimit: [null, []], - transportDeviceTelemetryDataPointsRateLimit: [null, []], - transportGatewayMsgRateLimit: [null, []], - transportGatewayTelemetryMsgRateLimit: [null, []], - transportGatewayTelemetryDataPointsRateLimit: [null, []], - transportGatewayDeviceMsgRateLimit: [null, []], - transportGatewayDeviceTelemetryMsgRateLimit: [null, []], - transportGatewayDeviceTelemetryDataPointsRateLimit: [null, []], - tenantEntityExportRateLimit: [null, []], - tenantEntityImportRateLimit: [null, []], - tenantNotificationRequestsRateLimit: [null, []], - tenantNotificationRequestsPerRuleRateLimit: [null, []], - maxTransportMessages: [null, [Validators.required, Validators.min(0)]], - maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], - maxREExecutions: [null, [Validators.required, Validators.min(0)]], - maxJSExecutions: [null, [Validators.required, Validators.min(0)]], - maxTbelExecutions: [null, [Validators.required, Validators.min(0)]], - maxDPStorageDays: [null, [Validators.required, Validators.min(0)]], - maxRuleNodeExecutionsPerMessage: [null, [Validators.required, Validators.min(0)]], - maxEmails: [null, [Validators.required, Validators.min(0)]], - maxSms: [null, []], - smsEnabled: [null, []], - maxCreatedAlarms: [null, [Validators.required, Validators.min(0)]], - maxDebugModeDurationMinutes: [null, [Validators.min(0)]], - defaultStorageTtlDays: [null, [Validators.required, Validators.min(0)]], - alarmsTtlDays: [null, [Validators.required, Validators.min(0)]], - rpcTtlDays: [null, [Validators.required, Validators.min(0)]], - queueStatsTtlDays: [null, [Validators.required, Validators.min(0)]], - ruleEngineExceptionsTtlDays: [null, [Validators.required, Validators.min(0)]], - tenantServerRestLimitsConfiguration: [null, []], - customerServerRestLimitsConfiguration: [null, []], - maxWsSessionsPerTenant: [null, [Validators.min(0)]], - maxWsSessionsPerCustomer: [null, [Validators.min(0)]], - maxWsSessionsPerRegularUser: [null, [Validators.min(0)]], - maxWsSessionsPerPublicUser: [null, [Validators.min(0)]], - wsMsgQueueLimitPerSession: [null, [Validators.min(0)]], - maxWsSubscriptionsPerTenant: [null, [Validators.min(0)]], - maxWsSubscriptionsPerCustomer: [null, [Validators.min(0)]], - maxWsSubscriptionsPerRegularUser: [null, [Validators.min(0)]], - maxWsSubscriptionsPerPublicUser: [null, [Validators.min(0)]], - wsUpdatesPerSessionRateLimit: [null, []], - cassandraWriteQueryTenantCoreRateLimits: [null, []], - cassandraReadQueryTenantCoreRateLimits: [null, []], - cassandraWriteQueryTenantRuleEngineRateLimits: [null, []], - cassandraReadQueryTenantRuleEngineRateLimits: [null, []], - edgeEventRateLimits: [null, []], - edgeEventRateLimitsPerEdge: [null, []], - edgeUplinkMessagesRateLimits: [null, []], - edgeUplinkMessagesRateLimitsPerEdge: [null, []], - maxCalculatedFieldsPerEntity: [null, [Validators.required, Validators.min(0)]], - maxArgumentsPerCF: [null, [Validators.required, Validators.min(0)]], - maxRelationLevelPerCfArgument: [null, [Validators.required, Validators.min(1)]], - minAllowedScheduledUpdateIntervalInSecForCF: [null, [Validators.required, Validators.min(0)]], - maxDataPointsPerRollingArg: [null, [Validators.required, Validators.min(0)]], - maxStateSizeInKBytes: [null, [Validators.required, Validators.min(0)]], - calculatedFieldDebugEventsRateLimit: [null, []], - maxSingleValueArgumentSizeInKBytes: [null, [Validators.required, Validators.min(0)]], + private propagateChange = (_v: any) => { }; + + constructor(private fb: FormBuilder) { + this.tenantProfileConfigurationForm = this.fb.group({ + maxDevices: [0, [Validators.required, Validators.min(0)]], + maxAssets: [0, [Validators.required, Validators.min(0)]], + maxCustomers: [0, [Validators.required, Validators.min(0)]], + maxUsers: [0, [Validators.required, Validators.min(0)]], + maxDashboards: [0, [Validators.required, Validators.min(0)]], + maxRuleChains: [0, [Validators.required, Validators.min(0)]], + maxEdges: [0, [Validators.required, Validators.min(0)]], + maxResourcesInBytes: [0, [Validators.required, Validators.min(0)]], + maxOtaPackagesInBytes: [0, [Validators.required, Validators.min(0)]], + maxResourceSize: [0, [Validators.required, Validators.min(0)]], + transportTenantMsgRateLimit: [''], + transportTenantTelemetryMsgRateLimit: [''], + transportTenantTelemetryDataPointsRateLimit: [''], + transportDeviceMsgRateLimit: [''], + transportDeviceTelemetryMsgRateLimit: [''], + transportDeviceTelemetryDataPointsRateLimit: [''], + transportGatewayMsgRateLimit: [''], + transportGatewayTelemetryMsgRateLimit: [''], + transportGatewayTelemetryDataPointsRateLimit: [''], + transportGatewayDeviceMsgRateLimit: [''], + transportGatewayDeviceTelemetryMsgRateLimit: [''], + transportGatewayDeviceTelemetryDataPointsRateLimit: [''], + tenantEntityExportRateLimit: [''], + tenantEntityImportRateLimit: [''], + tenantNotificationRequestsRateLimit: [''], + tenantNotificationRequestsPerRuleRateLimit: [''], + maxTransportMessages: [0, [Validators.required, Validators.min(0)]], + maxTransportDataPoints: [0, [Validators.required, Validators.min(0)]], + maxREExecutions: [0, [Validators.required, Validators.min(0)]], + maxJSExecutions: [0, [Validators.required, Validators.min(0)]], + maxTbelExecutions: [0, [Validators.required, Validators.min(0)]], + maxDPStorageDays: [0, [Validators.required, Validators.min(0)]], + maxRuleNodeExecutionsPerMessage: [0, [Validators.required, Validators.min(0)]], + maxEmails: [0, [Validators.required, Validators.min(0)]], + maxSms: [0], + smsEnabled: [false], + maxCreatedAlarms: [0, [Validators.required, Validators.min(0)]], + maxDebugModeDurationMinutes: [0, [Validators.min(0)]], + defaultStorageTtlDays: [0, [Validators.required, Validators.min(0)]], + alarmsTtlDays: [0, [Validators.required, Validators.min(0)]], + rpcTtlDays: [0, [Validators.required, Validators.min(0)]], + queueStatsTtlDays: [0, [Validators.required, Validators.min(0)]], + ruleEngineExceptionsTtlDays: [0, [Validators.required, Validators.min(0)]], + tenantServerRestLimitsConfiguration: [''], + customerServerRestLimitsConfiguration: [''], + maxWsSessionsPerTenant: [0, [Validators.min(0)]], + maxWsSessionsPerCustomer: [0, [Validators.min(0)]], + maxWsSessionsPerRegularUser: [0, [Validators.min(0)]], + maxWsSessionsPerPublicUser: [0, [Validators.min(0)]], + wsMsgQueueLimitPerSession: [0, [Validators.min(0)]], + maxWsSubscriptionsPerTenant: [0, [Validators.min(0)]], + maxWsSubscriptionsPerCustomer: [0, [Validators.min(0)]], + maxWsSubscriptionsPerRegularUser: [0, [Validators.min(0)]], + maxWsSubscriptionsPerPublicUser: [0, [Validators.min(0)]], + wsUpdatesPerSessionRateLimit: [''], + cassandraWriteQueryTenantCoreRateLimits: [''], + cassandraReadQueryTenantCoreRateLimits: [''], + cassandraWriteQueryTenantRuleEngineRateLimits: [''], + cassandraReadQueryTenantRuleEngineRateLimits: [''], + edgeEventRateLimits: [''], + edgeEventRateLimitsPerEdge: [''], + edgeUplinkMessagesRateLimits: [''], + edgeUplinkMessagesRateLimitsPerEdge: [''], + maxCalculatedFieldsPerEntity: [0, [Validators.required, Validators.min(0)]], + maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], + maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], + maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], + minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], + maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], + maxStateSizeInKBytes: [0, [Validators.required, Validators.min(0)]], + calculatedFieldDebugEventsRateLimit: [''], + maxSingleValueArgumentSizeInKBytes: [0, [Validators.required, Validators.min(0)]], }); - this.defaultTenantProfileConfigurationFormGroup.get('smsEnabled').valueChanges.pipe( - takeUntil(this.destroy$) + this.tenantProfileConfigurationForm.get('smsEnabled').valueChanges.pipe( + takeUntilDestroyed() ).subscribe((value: boolean) => { this.maxSmsValidation(value); } ); - this.defaultTenantProfileConfigurationFormGroup.valueChanges.pipe( - takeUntil(this.destroy$) + this.tenantProfileConfigurationForm.valueChanges.pipe( + takeUntilDestroyed() ).subscribe(() => { this.updateModel(); }); @@ -148,48 +140,40 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA private maxSmsValidation(smsEnabled: boolean) { if (smsEnabled) { - this.defaultTenantProfileConfigurationFormGroup.get('maxSms').addValidators([Validators.required, Validators.min(0)]); + this.tenantProfileConfigurationForm.get('maxSms').addValidators([Validators.required, Validators.min(0)]); } else { - this.defaultTenantProfileConfigurationFormGroup.get('maxSms').clearValidators(); + this.tenantProfileConfigurationForm.get('maxSms').clearValidators(); } - this.defaultTenantProfileConfigurationFormGroup.get('maxSms').updateValueAndValidity({emitEvent: false}); - } - - ngOnDestroy(): void { - this.destroy$.next(); - this.destroy$.complete(); + this.tenantProfileConfigurationForm.get('maxSms').updateValueAndValidity({emitEvent: false}); } registerOnChange(fn: any): void { this.propagateChange = fn; } - registerOnTouched(fn: any): void { - } - - ngOnInit() { + registerOnTouched(_fn: any): void { } setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; if (this.disabled) { - this.defaultTenantProfileConfigurationFormGroup.disable({emitEvent: false}); + this.tenantProfileConfigurationForm.disable({emitEvent: false}); } else { - this.defaultTenantProfileConfigurationFormGroup.enable({emitEvent: false}); + this.tenantProfileConfigurationForm.enable({emitEvent: false}); } } writeValue(value: DefaultTenantProfileConfiguration | null): void { if (isDefinedAndNotNull(value)) { this.maxSmsValidation(value.smsEnabled); - this.defaultTenantProfileConfigurationFormGroup.patchValue(value, {emitEvent: false}); + this.tenantProfileConfigurationForm.patchValue(value, {emitEvent: false}); } } private updateModel() { - let configuration: TenantProfileConfiguration = null; - if (this.defaultTenantProfileConfigurationFormGroup.valid) { - configuration = this.defaultTenantProfileConfigurationFormGroup.getRawValue(); + let configuration: DefaultTenantProfileConfiguration = null; + if (this.tenantProfileConfigurationForm.valid) { + configuration = this.tenantProfileConfigurationForm.getRawValue(); } this.propagateChange(configuration); } diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 059fe39ada..1ed1092207 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -19,6 +19,11 @@ import { TenantId } from './id/tenant-id'; import { TenantProfileId } from '@shared/models/id/tenant-profile-id'; import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { QueueInfo } from '@shared/models/queue.models'; +import { FormControl } from '@angular/forms'; + +export type FormControlsFrom = { + [K in keyof T]-?: FormControl; +}; export enum TenantProfileType { DEFAULT = 'DEFAULT' @@ -101,6 +106,9 @@ export interface DefaultTenantProfileConfiguration { maxCalculatedFieldsPerEntity: number; maxArgumentsPerCF: number; + maxRelationLevelPerCfArgument: number; + maxRelatedEntitiesToReturnPerCfArgument: number; + minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; maxStateSizeInKBytes: number; maxSingleValueArgumentSizeInKBytes: number; @@ -165,6 +173,9 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxCalculatedFieldsPerEntity: 5, maxArgumentsPerCF: 10, maxDataPointsPerRollingArg: 1000, + maxRelationLevelPerCfArgument: 10, + maxRelatedEntitiesToReturnPerCfArgument: 100, + minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, maxSingleValueArgumentSizeInKBytes: 2, calculatedFieldDebugEventsRateLimit: '' diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 7f728c0b63..3ee738a8b0 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5952,6 +5952,10 @@ "ws-limit-max-subscriptions-per-regular-user": "Subscriptions per regular user maximum number", "ws-limit-max-subscriptions-per-public-user": "Subscriptions per public user maximum number", "ws-limit-updates-per-session": "WS updates per session", + "relation-search-entity-limit": "Relation search entity limit", + "relation-search-entity-limit-hint": "Limits the number of entities resolved at the last level of the relation path. Applies to 'Related entities' arguments and Propagation fields.", + "relation-search-entity-limit-required": "Relation search entity limit", + "relation-search-entity-limit-range": "Relation search entity limit can't be less than '1'", "rate-limits": { "add-limit": "Add limit", "and-also-less-than": "and also less than", From f951c64767c504142bdaaab1ad69711d995ec73c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 11:14:07 +0300 Subject: [PATCH 054/122] added state to proto and fixed deduplication logic --- ...ValuesAggregationCalculatedFieldState.java | 23 +++---- .../state/aggregation/function/new_agg.json | 60 ------------------- .../server/utils/CalculatedFieldUtils.java | 16 +++-- ...tValuesAggregationCalculatedFieldTest.java | 16 ++++- common/proto/src/main/proto/queue.proto | 8 ++- 5 files changed, 42 insertions(+), 81 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 6ae023ed1e..f3560faafa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -106,21 +106,22 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); - if (!shouldRecalculate() && !shouldRecalculate) { + boolean cfUpdated = updatedArgs != null && updatedArgs.isEmpty(); + if (shouldRecalculate() || cfUpdated) { + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); } - Output output = ctx.getOutput(); - ObjectNode aggResult = aggregateMetrics(output); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) - .build()); } private boolean shouldRecalculate() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json deleted file mode 100644 index c6b841b673..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET", - "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "arguments": { - "oc": { - "refEntityKey": { - "key": "occupied", - "type": "TS_LATEST" - }, - "defaultValue": "false" - } - }, - "deduplicationIntervalMillis": 20000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 00f4cbde0f..f8a7d17543 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -35,6 +35,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; +import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,13 +97,11 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { - builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - } + LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { @@ -120,6 +119,10 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + builder.setLatestValuesAggregationState(aggBuilder.build()); + } return builder.build(); } @@ -237,15 +240,16 @@ public class CalculatedFieldUtils { } case LATEST_VALUES_AGGREGATION -> { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); - proto.getAggArgumentsList().forEach(argProto -> { + aggregationStateProto.getAggArgumentsList().forEach(argProto -> { AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); - aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 8c2c2ca68c..46b8b78d57 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -489,10 +489,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); + }); + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -544,9 +550,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); saveCalculatedField(cf); + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1e01916e55..b59e01128b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -920,6 +920,11 @@ message AggSingleArgumentEntryProto { SingleValueArgumentProto value = 2; } +message LatestValuesAggregationStateProto { + int64 lastArgsUpdateTs = 1; + repeated AggSingleArgumentEntryProto aggArguments = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -927,8 +932,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - repeated AggSingleArgumentEntryProto aggArguments = 7; - int64 lastArgsUpdateTs = 8; + LatestValuesAggregationStateProto latestValuesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 43004ff5d87873d9c1b1dfc496f00918b471edad Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 17 Oct 2025 14:48:17 +0300 Subject: [PATCH 055/122] Alarm rules CF: improve alarm action handling --- .../cf/ctx/state/alarm/AlarmCalculatedFieldState.java | 6 ++++-- .../thingsboard/server/common/data/msg/TbMsgTypeTest.java | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index c140ba78af..61d55f376f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -223,6 +223,9 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { private void processAlarmClear(Alarm alarm) { currentAlarm = null; createRuleStates.values().forEach(AlarmRuleState::clear); + createRuleStates.clear(); + clearState(clearRuleState); + clearRuleState = null; } private void processAlarmAck(Alarm alarm) { @@ -231,8 +234,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } private void processAlarmDelete(Alarm alarm) { - currentAlarm = null; - createRuleStates.values().forEach(AlarmRuleState::clear); + processAlarmClear(alarm); } private TbAlarmResult createOrClearAlarms(Function evalFunction, diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/msg/TbMsgTypeTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/msg/TbMsgTypeTest.java index 563c6015ef..54238c8751 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/msg/TbMsgTypeTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/msg/TbMsgTypeTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; -import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM; import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM_DELETE; import static org.thingsboard.server.common.data.msg.TbMsgType.DEDUPLICATION_TIMEOUT_SELF_MSG; import static org.thingsboard.server.common.data.msg.TbMsgType.DELAY_TIMEOUT_SELF_MSG; @@ -39,7 +38,6 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.SEND_EMAIL; class TbMsgTypeTest { private static final List typesWithNullRuleNodeConnection = List.of( - ALARM, ALARM_DELETE, ENTITY_ASSIGNED_TO_EDGE, ENTITY_UNASSIGNED_FROM_EDGE, From 8af42148b60eb2457c78609c71baf89e11244ca5 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 17 Oct 2025 15:45:56 +0300 Subject: [PATCH 056/122] Alarm rules CF: push different alarm msg types --- .../server/service/cf/AlarmCalculatedFieldResult.java | 7 ++++++- .../org/thingsboard/server/common/data/msg/TbMsgType.java | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java index 61f9cf37ff..498a215e17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AlarmCalculatedFieldResult.java @@ -38,15 +38,20 @@ public class AlarmCalculatedFieldResult implements CalculatedFieldResult { @Override public TbMsg toTbMsg(EntityId entityId, List cfIds) { + TbMsgType msgType; TbMsgMetaData metaData = new TbMsgMetaData(); if (alarmResult.isCreated()) { + msgType = TbMsgType.ALARM_CREATED; metaData.putValue(DataConstants.IS_NEW_ALARM, Boolean.TRUE.toString()); } else if (alarmResult.isUpdated()) { + msgType = TbMsgType.ALARM_UPDATED; metaData.putValue(DataConstants.IS_EXISTING_ALARM, Boolean.TRUE.toString()); } else if (alarmResult.isSeverityUpdated()) { + msgType = TbMsgType.ALARM_SEVERITY_UPDATED; metaData.putValue(DataConstants.IS_EXISTING_ALARM, Boolean.TRUE.toString()); metaData.putValue(DataConstants.IS_SEVERITY_UPDATED_ALARM, Boolean.TRUE.toString()); } else { + msgType = TbMsgType.ALARM_CLEAR; metaData.putValue(DataConstants.IS_CLEARED_ALARM, Boolean.TRUE.toString()); } if (alarmResult.getConditionRepeats() != null) { @@ -57,7 +62,7 @@ public class AlarmCalculatedFieldResult implements CalculatedFieldResult { } return TbMsg.newMsg() - .type(TbMsgType.ALARM) + .type(msgType) .originator(entityId) .data(JacksonUtil.toString(alarmResult.getAlarm())) .metaData(metaData) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java b/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java index e2949c96eb..720dc5b790 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/msg/TbMsgType.java @@ -39,6 +39,9 @@ public enum TbMsgType { ATTRIBUTES_UPDATED("Attributes Updated"), ATTRIBUTES_DELETED("Attributes Deleted"), ALARM("Alarm"), + ALARM_CREATED("Alarm Created"), + ALARM_UPDATED("Alarm Updated"), + ALARM_SEVERITY_UPDATED("Alarm Severity Updated"), ALARM_ACK("Alarm Acknowledged"), ALARM_CLEAR("Alarm Cleared"), ALARM_DELETE, From 6a307041b699f4d2a5d1c27e8755b8538f0fec31 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 15:55:44 +0300 Subject: [PATCH 057/122] added tests for entry --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 24 ++-- .../ctx/state/BaseCalculatedFieldState.java | 3 +- .../state/aggregation/AggArgumentEntry.java | 7 +- .../AggSingleEntityArgumentEntry.java | 30 +++-- .../cf/ctx/state/AggArgumentEntryTest.java | 112 ++++++++++++++++++ .../AggSingleEntityArgumentEntryTest.java | 101 ++++++++++++++++ .../server/dao/relation/RelationService.java | 2 + .../data/cf/configuration/Argument.java | 4 + ...gregationCalculatedFieldConfiguration.java | 13 ++ .../dao/relation/BaseRelationService.java | 15 +++ 12 files changed, 288 insertions(+), 31 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ce29dc06e0..58113deb08 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -67,7 +67,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -686,9 +685,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map.Entry::getKey, argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) )); - } else { - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); } + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 94695bcc8c..7f47dba2ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -332,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -344,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 37c7b2ef6c..147882c3c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -37,8 +37,9 @@ import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,20 +192,15 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byToAndType.isEmpty()) { - return true; - } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byFromAndType.isEmpty()) { - return true; - } - } + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (!byRelationPathQuery.isEmpty()) { + return true; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 2d853fd3fd..ee10857e6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -73,7 +74,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || newEntry.isForceResetPrevious()) { + if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 7e3a8623e4..e002aece2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -52,7 +52,12 @@ public class AggArgumentEntry implements ArgumentEntry { if (aggSingleEntityArgumentEntry.isDeleted()) { aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + } else { + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + } } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 32ce77311d..6430fe3f1c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -62,25 +62,35 @@ public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() <= ts) { - return false; + if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { + if (aggSingleEntityEntry.isForceResetPrevious()) { + return applyNewEntry(aggSingleEntityEntry); } - Long newVersion = singleValueEntry.getVersion(); + if (aggSingleEntityEntry.getTs() < this.ts) { + if (!isDefaultValue()) { + return false; + } + } + + Long newVersion = aggSingleEntityEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { - this.ts = singleValueEntry.getTs(); - this.version = newVersion; - this.kvEntryValue = singleValueEntry.getKvEntryValue(); - this.entityId = singleValueEntry.getEntityId(); - return true; + return applyNewEntry(aggSingleEntityEntry); } } else { - throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); } return false; } + private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.kvEntryValue = entry.getKvEntryValue(); + this.entityId = entry.getEntityId(); + return true; + } + @Override public ArgumentEntryType getType() { return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java new file mode 100644 index 0000000000..c1c2d85c55 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggArgumentEntryTest { + + private AggArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + Map aggInputs = new HashMap<>(); + aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + + entry = new AggArgumentEntry(aggInputs, false); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenAggArgumentEntryPasser() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + DeviceId device4 = new DeviceId(UUID.randomUUID()); + + AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + ), false); + + assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(4); + assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(3); + assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(2); + assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(1); + assertThat(aggInputs.get(device2)).isNull(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java new file mode 100644 index 0000000000..0401bb0156 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggSingleEntityArgumentEntryTest { + + private AggSingleEntityArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenResetPrevious() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); + singleEntityArgumentEntry.setForceResetPrevious(true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); + assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); + assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); + } + + + @Test + void testUpdateEntryWithTheSameTsAndVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); + } + + @Test + void testUpdateEntryWithTheSameTsAndDifferentVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(56L); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(76L); + assertThat(entry.getVersion()).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); + } + + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); + } + + @Test + void testUpdateEntryWithOldTs() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..214dc5247e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,8 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 4de4551e73..8d4a831cf9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -45,4 +45,8 @@ public class Argument { return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_OWNER; } + public boolean hasTsRollingArgument() { + return ArgumentType.TS_ROLLING.equals(refEntityKey.getType()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 43a3360ce6..721930e676 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -41,6 +41,19 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public void validate() { + if (relation == null) { + throw new IllegalArgumentException("Relation must be specified!"); + } + relation.validate(); + if (arguments.containsKey("ctx")) { + throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); + } + if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); + } + if (metrics.isEmpty()) { + throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); + } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..ec203e5309 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,21 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + if (relationPathQuery.levels().size() == 1) { + RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + case TO -> findByToAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + }; + } + return relationDao.findByRelationPathQuery(tenantId, relationPathQuery); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); From 37039a995dca6c3fa05395992ce335d4b579083a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 10:02:55 +0300 Subject: [PATCH 058/122] added minDeduplicationInterval to tenant profile config --- .../main/data/upgrade/basic/schema_update.sql | 8 +++ .../controller/SystemInfoController.java | 1 + .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...ValuesAggregationCalculatedFieldState.java | 2 +- ...tValuesAggregationCalculatedFieldTest.java | 64 ++++++++++--------- .../server/common/data/SystemParams.java | 1 + ...gregationCalculatedFieldConfiguration.java | 2 +- .../DefaultTenantProfileConfiguration.java | 2 + .../CalculatedFieldDataValidator.java | 15 ++++- 9 files changed, 63 insertions(+), 34 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 495aee00e2..c5b73c6899 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -34,6 +34,12 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' THEN NULL ELSE to_jsonb(10) + END, + 'minAllowedDeduplicationIntervalInSecForCF', + CASE + WHEN (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' + THEN NULL + ELSE to_jsonb(3600) END ) ), @@ -43,6 +49,8 @@ WHERE NOT ( (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF' AND (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' + AND + (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index b9968aefa9..82807d0762 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -164,6 +164,7 @@ public class SystemInfoController extends BaseController { systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); systemParams.setMinAllowedScheduledUpdateIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedScheduledUpdateIntervalInSecForCF()); systemParams.setMaxRelationLevelPerCfArgument(tenantProfileConfiguration.getMaxRelationLevelPerCfArgument()); + systemParams.setMinAllowedDeduplicationIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedDeduplicationIntervalInSecForCF()); systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f69d611b64..1d9ccf10d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,7 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { + && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index f3560faafa..fdc045a0db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -66,7 +66,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF super.setCtx(ctx, actorCtx); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @Override diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 46b8b78d57..e8c244b57e 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -79,12 +79,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private long deduplicationInterval = 10000; + private long deduplicationInterval = 10; @Before public void beforeTest() throws Exception { loginSysAdmin(); + updateDefaultTenantProfileConfig(tenantProfileConfig -> { + tenantProfileConfig.setMinAllowedDeduplicationIntervalInSecForCF(1); + }); + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); savedTenant = saveTenant(tenant); @@ -131,7 +135,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -149,7 +153,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -171,7 +175,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Asset asset2 = createAsset("Asset 2", assetProfile.getId()); - await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); @@ -184,7 +188,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -196,7 +200,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -218,7 +222,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -240,7 +244,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -262,7 +266,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -280,7 +284,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -301,7 +305,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -319,13 +323,13 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); postTelemetry(device2.getId(), "{\"occupied\":false}"); - await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -355,7 +359,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -368,7 +372,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); - await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -390,7 +394,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -408,7 +412,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -432,7 +436,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); - await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -458,7 +462,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setArguments(Map.of("oc", argument)); saveCalculatedField(cf); - await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -475,7 +479,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -489,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); @@ -498,7 +502,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -511,7 +515,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -524,7 +528,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setOutput(output); saveCalculatedField(cf); - await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); @@ -540,17 +544,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); - configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); - await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -558,7 +562,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); @@ -566,7 +570,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } private void checkInitialCalculation() { - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); } @@ -656,7 +660,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); - configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setDeduplicationIntervalInSec(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index f83a812529..6a475daae3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -40,5 +40,6 @@ public class SystemParams { long maxDataPointsPerRollingArg; int minAllowedScheduledUpdateIntervalInSecForCF; int maxRelationLevelPerCfArgument; + long minAllowedDeduplicationIntervalInSecForCF; TrendzSettings trendzSettings; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 721930e676..5f8fb65265 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -29,7 +29,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private RelationPathLevel relation; private Map arguments; - private long deduplicationIntervalMillis; + private long deduplicationIntervalInSec; private Map metrics; private Output output; private boolean useLatestTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index c6bd9a7f38..9f124fd9e4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -184,6 +184,8 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxStateSizeInKBytes = 32; @Schema(example = "2") private long maxSingleValueArgumentSizeInKBytes = 2; + @Schema(example = "3600") + private long minAllowedDeduplicationIntervalInSecForCF = 3600; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 67ed8f29b0..319a4fbe8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -46,6 +47,7 @@ public class CalculatedFieldDataValidator extends DataValidator validateCalculatedFieldConfiguration(calculatedField); validateSchedulingConfiguration(tenantId, calculatedField); validateRelationQuerySourceArguments(tenantId, calculatedField); + validateAggregationConfiguration(tenantId, calculatedField); } @Override @@ -87,7 +89,7 @@ public class CalculatedFieldDataValidator extends DataValidator private void validateSchedulingConfiguration(TenantId tenantId, CalculatedField calculatedField) { if (!(calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledUpdateCfg) - || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { + || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { return; } long minAllowedScheduledUpdateInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedScheduledUpdateIntervalInSecForCF); @@ -110,6 +112,17 @@ public class CalculatedFieldDataValidator extends DataValidator wrapAsDataValidation(() -> relationQueryDynamicSourceConfiguration.validateMaxRelationLevel(argumentName, maxRelationLevel))); } + private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { + if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + return; + } + long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); + if (aggConfiguration.getDeduplicationIntervalInSec() < minAllowedDeduplicationInterval) { + throw new IllegalArgumentException("Deduplication interval is less than configured " + + "minimum allowed interval in tenant profile: " + minAllowedDeduplicationInterval); + } + } + private static void wrapAsDataValidation(Runnable validation) { try { validation.run(); From 55fb64ef13af8c7cd84033ce31639087f4cf8ac7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 11:20:38 +0300 Subject: [PATCH 059/122] fixed delete attributes --- ...CalculatedFieldEntityMessageProcessor.java | 49 +++++++----- ...alculatedFieldManagerMessageProcessor.java | 30 ++++---- ...tValuesAggregationCalculatedFieldTest.java | 75 +++++++++++++++++++ 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 58113deb08..e0faf46185 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -551,19 +551,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), Collections.emptyMap(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map relatedEntityArgs, List data) { Map arguments = new HashMap<>(); - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } @@ -587,17 +587,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getRelatedEntityArguments(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); + Map relatedEntityArgs = ctx.getRelatedEntityArguments(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, relatedEntityArgs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map relatedEntityArgs, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -613,10 +613,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM arguments.put(argName, new SingleValueArgumentEntry(item)); } } - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } @@ -627,26 +627,40 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { + Map relatedEntityArguments = ctx.getRelatedEntityArguments(); + if (argNames.isEmpty() && relatedEntityArguments.isEmpty()) { return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); - return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedEntityArguments, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, Map argNames, Map configArguments, List geofencingArgNames, - List relatedEntityArgNames, + Map relatedEntityArgs, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); + if (!relatedEntityArgs.isEmpty()) { + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + if (relatedEntityArgs.containsKey(key)) { + String argName = relatedEntityArgs.get(key); + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry(); + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + } + } + } for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -662,12 +676,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - if (relatedEntityArgNames.contains(argName)) { - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); - continue; - } arguments.put(argName, argumentEntry); - } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index af5a672157..24fd5d320b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -41,9 +41,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -520,22 +520,22 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byToAndType != null && !byToAndType.isEmpty()) { - EntityRelation entityRelation = byToAndType.get(0); // only one supported + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; + }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (byRelationPathQuery != null && !byRelationPathQuery.isEmpty()) { + switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = byRelationPathQuery.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byFromAndType != null && !byFromAndType.isEmpty()) { - for (EntityRelation entityRelation : byFromAndType) { - if (entityRelation.getTo().equals(cf.getEntityId())) { - result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); - } - } + case TO -> { + byRelationPathQuery.stream() + .filter(entityRelation -> entityRelation.getTo().equals(cf.getEntityId())) + .forEach(entityRelation -> result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo()))); } } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index e8c244b57e..eb11a329be 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -383,6 +383,44 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testDeleteAttr_checkAggregationWithDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + postAttributes(device3.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + postAttributes(device4.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + + createOccupancyCFWithAttr(asset2.getId()); + + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + + await().alias("delete attribute and perform aggregation with default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + @Test public void testCreateRelation_checkAggregation() throws Exception { createOccupancyCF(asset.getId()); @@ -646,6 +684,43 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output); } + private CalculatedField createOccupancyCFWithAttr(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false;"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true;"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Occupied spaces", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createAggCf(String name, EntityId entityId, RelationPathLevel relation, From a0b38a7eb4731ecce9f5dbef81d8b5ae1a87b73e Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 20 Oct 2025 13:09:10 +0300 Subject: [PATCH 060/122] Fix NotificationRuleApiTest --- .../alarm/AlarmCalculatedFieldState.java | 2 - .../notification/NotificationRuleApiTest.java | 91 ++++++++----------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 61d55f376f..e36892378c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -189,8 +189,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) { initCurrentAlarm(ctx); - // FIXME: don't create alarm if attrs were deleted, or config is updated - // TODO: what if expression is changed? do we reevaluate? or only on new events? TbAlarmResult result = createOrClearAlarms(state -> { if (updatedArgs != null) { boolean newEvent = !updatedArgs.isEmpty(); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index bab70ea505..2bfdc2f330 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -27,7 +27,7 @@ import org.springframework.data.util.Pair; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cache.limits.RateLimitService; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; @@ -39,17 +39,19 @@ import org.thingsboard.server.common.data.alarm.AlarmCommentType; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.alarm.rule.AlarmRule; +import org.thingsboard.server.common.data.alarm.rule.condition.SimpleAlarmCondition; +import org.thingsboard.server.common.data.alarm.rule.condition.expression.TbelAlarmConditionExpression; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; -import org.thingsboard.server.common.data.device.profile.AlarmCondition; -import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; -import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; -import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; -import org.thingsboard.server.common.data.device.profile.AlarmRule; -import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; -import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpec; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.DeviceId; @@ -87,9 +89,6 @@ import org.thingsboard.server.common.data.notification.targets.platform.SystemAd import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.query.BooleanFilterPredicate; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.security.Authority; @@ -106,12 +105,10 @@ import org.thingsboard.server.service.system.DefaultSystemInfoService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.lang.reflect.Method; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -193,7 +190,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRuleProcessing_alarmTrigger() throws Exception { String notificationSubject = "Alarm type: ${alarmType}, status: ${alarmStatus}, " + - "severity: ${alarmSeverity}, deviceId: ${alarmOriginatorId}"; + "severity: ${alarmSeverity}, deviceId: ${alarmOriginatorId}"; String notificationText = "Status: ${alarmStatus}, severity: ${alarmSeverity}"; NotificationTemplate notificationTemplate = createNotificationTemplate(NotificationType.ALARM, notificationSubject, notificationText, NotificationDeliveryMethod.WEB); @@ -234,8 +231,8 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { }); JsonNode attr = JacksonUtil.newObjectNode() - .set("bool", BooleanNode.TRUE); - doPost("/api/plugins/telemetry/" + device.getId() + "/" + DataConstants.SHARED_SCOPE, attr); + .set("createAlarm", BooleanNode.TRUE); + postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, attr.toString()); await().atMost(10, TimeUnit.SECONDS) .until(() -> alarmSubscriptionService.findLatestByOriginatorAndType(tenantId, device.getId(), alarmType) != null); @@ -250,7 +247,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { assertThat(actualDelay).isCloseTo(expectedDelay, offset(2.0)); assertThat(notification.getSubject()).isEqualTo("Alarm type: " + alarmType + ", status: " + AlarmStatus.ACTIVE_UNACK + ", " + - "severity: " + AlarmSeverity.CRITICAL.toString().toLowerCase() + ", deviceId: " + device.getId()); + "severity: " + AlarmSeverity.CRITICAL.toString().toLowerCase() + ", deviceId: " + device.getId()); assertThat(notification.getText()).isEqualTo("Status: " + AlarmStatus.ACTIVE_UNACK + ", severity: " + AlarmSeverity.CRITICAL.toString().toLowerCase()); assertThat(notification.getType()).isEqualTo(NotificationType.ALARM); @@ -270,7 +267,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { wsClient.waitForUpdate(true); Notification updatedNotification = wsClient.getLastDataUpdate().getUpdate(); assertThat(updatedNotification.getSubject()).isEqualTo("Alarm type: " + alarmType + ", status: " + expectedStatus + ", " + - "severity: " + expectedSeverity.toString().toLowerCase() + ", deviceId: " + device.getId()); + "severity: " + expectedSeverity.toString().toLowerCase() + ", deviceId: " + device.getId()); assertThat(updatedNotification.getText()).isEqualTo("Status: " + expectedStatus + ", severity: " + expectedSeverity.toString().toLowerCase()); wsClient.close(); @@ -296,7 +293,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { List notifications = getMyNotifications(false, 10); assertThat(notifications).singleElement().matches(notification -> { return notification.getType() == NotificationType.ALARM && - notification.getSubject().equals("New alarm 'testAlarm'"); + notification.getSubject().equals("New alarm 'testAlarm'"); }); }); } @@ -341,8 +338,8 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { getWsClient().subscribeForUnreadNotifications(10).waitForReply(true); getWsClient().registerWaitForUpdate(); JsonNode attr = JacksonUtil.newObjectNode() - .set("bool", BooleanNode.TRUE); - doPost("/api/plugins/telemetry/" + device.getId() + "/" + DataConstants.SHARED_SCOPE, attr); + .set("createAlarm", BooleanNode.TRUE); + postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, attr.toString()); await().atMost(10, TimeUnit.SECONDS) .until(() -> alarmSubscriptionService.findLatestByOriginatorAndType(tenantId, device.getId(), alarmType) != null); @@ -491,11 +488,11 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { }); assertThat(notifications).anySatisfy(notification -> { assertThat(notification.getText()).isEqualTo("Rate limits for REST API requests per customer " + - "exceeded for 'Customer'"); + "exceeded for 'Customer'"); }); assertThat(notifications).anySatisfy(notification -> { assertThat(notification.getText()).isEqualTo("Rate limits for notification requests " + - "per rule exceeded for '" + rule.getName() + "'"); + "per rule exceeded for '" + rule.getName() + "'"); }); loginSysAdmin(); @@ -748,7 +745,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { .build(); assertThat(DefaultNotificationDeduplicationService.getDeduplicationKey(expectedTrigger, rule)) .isEqualTo("RATE_LIMITS:TENANT:" + tenantId + ":ENTITY_EXPORT_" + - target.getId() + ":ENTITY_EXPORT,TRANSPORT_MESSAGES_PER_DEVICE"); + target.getId() + ":ENTITY_EXPORT,TRANSPORT_MESSAGES_PER_DEVICE"); loginTenantAdmin(); getWsClient().subscribeForUnreadNotifications(10).waitForReply(); @@ -944,35 +941,27 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { private DeviceProfile createDeviceProfileWithAlarmRules(String alarmType) { DeviceProfile deviceProfile = createDeviceProfile("For notification rule test"); deviceProfile.setTenantId(tenantId); + deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); - List alarms = new ArrayList<>(); - DeviceProfileAlarm alarm = new DeviceProfileAlarm(); - alarm.setAlarmType(alarmType); - alarm.setId(alarmType); + CalculatedField alarmCf = new CalculatedField(); + alarmCf.setType(CalculatedFieldType.ALARM); + alarmCf.setEntityId(deviceProfile.getId()); + alarmCf.setName(alarmType); + AlarmCalculatedFieldConfiguration configuration = new AlarmCalculatedFieldConfiguration(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("createAlarm", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + configuration.setArguments(Map.of("createAlarm", argument)); AlarmRule alarmRule = new AlarmRule(); - alarmRule.setAlarmDetails("Details"); - AlarmCondition alarmCondition = new AlarmCondition(); - alarmCondition.setSpec(new SimpleAlarmConditionSpec()); - List condition = new ArrayList<>(); - - AlarmConditionFilter alarmConditionFilter = new AlarmConditionFilter(); - alarmConditionFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "bool")); - BooleanFilterPredicate predicate = new BooleanFilterPredicate(); - predicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); - predicate.setValue(new FilterPredicateValue<>(true)); - - alarmConditionFilter.setPredicate(predicate); - alarmConditionFilter.setValueType(EntityKeyValueType.BOOLEAN); - condition.add(alarmConditionFilter); - alarmCondition.setCondition(condition); - alarmRule.setCondition(alarmCondition); - TreeMap createRules = new TreeMap<>(); - createRules.put(AlarmSeverity.CRITICAL, alarmRule); - alarm.setCreateRules(createRules); - alarms.add(alarm); - - deviceProfile.getProfileData().setAlarms(alarms); - deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + SimpleAlarmCondition condition = new SimpleAlarmCondition(); + TbelAlarmConditionExpression expression = new TbelAlarmConditionExpression(); + expression.setExpression("return createAlarm == true;"); + condition.setExpression(expression); + alarmRule.setCondition(condition); + configuration.setCreateRules(Map.of( + AlarmSeverity.CRITICAL, alarmRule + )); + alarmCf.setConfiguration(configuration); + saveCalculatedField(alarmCf); return deviceProfile; } From 663b69fb706354b6365c7eff76b074f0be0b9c0f Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 20 Oct 2025 13:11:16 +0300 Subject: [PATCH 061/122] Fix findByEntityIdAndTypeAndName for CF --- .../server/dao/sql/cf/CalculatedFieldRepository.java | 3 +-- .../thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 9a1f904788..d4e471b838 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -19,7 +19,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -30,7 +29,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index e0e5ef60c4..2b29d1afcc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -69,7 +69,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao Date: Mon, 20 Oct 2025 13:19:56 +0300 Subject: [PATCH 062/122] TestRestClient code clean up --- .../src/test/java/org/thingsboard/server/msa/TestRestClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 7bca833bf4..7a4d3f1cfe 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -415,7 +415,6 @@ public class TestRestClient { queryParams.put("toType", toId.getEntityType().name()); return given().spec(requestSpec) .queryParams(queryParams) - //.delete("/api/v2/relation?fromId={fromId}&fromType={fromType}&relationType={relationType}&toId={toId}&toType={toType}") .delete("/api/v2/relation") .then() .statusCode(HTTP_OK) From fab3cfbc863e06a68d4cf3dd3d4570dd46ce2d2f Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Mon, 20 Oct 2025 14:16:28 +0300 Subject: [PATCH 063/122] Alarm rules CF: add test for manual alarm clear --- .../alarm/AlarmCalculatedFieldState.java | 4 +- .../thingsboard/server/cf/AlarmRulesTest.java | 47 ++++++++++++++++--- .../src/test/resources/logback-test.xml | 2 + 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index e36892378c..02f1725cf2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -220,10 +220,8 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { private void processAlarmClear(Alarm alarm) { currentAlarm = null; - createRuleStates.values().forEach(AlarmRuleState::clear); - createRuleStates.clear(); + createRuleStates.values().forEach(this::clearState); clearState(clearRuleState); - clearRuleState = null; } private void processAlarmAck(Alarm alarm) { diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index ae0010ea57..2b43373ee5 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; @@ -689,16 +690,49 @@ public class AlarmRulesTest extends AbstractControllerTest { }); } + @Test + public void testManualClearAlarm() throws Exception { + Argument temperatureArgument = new Argument(); + temperatureArgument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + temperatureArgument.setDefaultValue("0"); + Map arguments = Map.of( + "temperature", temperatureArgument + ); + + Map createRules = Map.of( + AlarmSeverity.CRITICAL, new Condition("return temperature >= 50;", null, null) + ); + + CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature Alarm", + arguments, createRules, null); + + postTelemetry(deviceId, "{\"temperature\":50}"); + Alarm alarm = checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }).getAlarm(); + + doPost("/api/alarm/" + alarm.getId() + "/clear", AlarmInfo.class); + Thread.sleep(1000); + postTelemetry(deviceId, "{\"temperature\":50}"); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.getAlarm().getId()).isNotEqualTo(alarm.getId()); + assertThat(alarmResult.isCreated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); + }); + } + // TODO: MSA tests - // TODO: test when attribute or telemetry is deleted without default value - perform calculation not happens - private void checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { - checkAlarmResult(calculatedField, null, assertion); + private TbAlarmResult checkAlarmResult(CalculatedField calculatedField, Consumer assertion) { + return checkAlarmResult(calculatedField, null, assertion); } - private void checkAlarmResult(CalculatedField calculatedField, - Predicate waitFor, - Consumer assertion) { + private TbAlarmResult checkAlarmResult(CalculatedField calculatedField, + Predicate waitFor, + Consumer assertion) { TbAlarmResult alarmResult = await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getLatestAlarmResult(calculatedField.getId()), result -> result != null && (waitFor == null || waitFor.test(result))); @@ -707,6 +741,7 @@ public class AlarmRulesTest extends AbstractControllerTest { Alarm alarm = alarmResult.getAlarm(); assertThat(alarm.getOriginator()).isEqualTo(originatorId); assertThat(alarm.getType()).isEqualTo(calculatedField.getName()); + return alarmResult; } private TbAlarmResult getLatestAlarmResult(CalculatedFieldId calculatedFieldId) { diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 13c93da411..56dbbfc125 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -17,6 +17,8 @@ + + From 5e8de6b955f432a244dbb5c4406bc970ee18c5a8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 10:33:20 +0300 Subject: [PATCH 064/122] renamed cf --- ...CalculatedFieldEntityMessageProcessor.java | 12 +++++----- ...alculatedFieldManagerMessageProcessor.java | 12 +++++----- ...tractCalculatedFieldProcessingService.java | 8 +++---- .../cf/DefaultCalculatedFieldCache.java | 6 ++--- .../DefaultCalculatedFieldQueueService.java | 4 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 22 +++++++++---------- ...itiesAggregationCalculatedFieldState.java} | 10 ++++----- .../utils/CalculatedFieldArgumentUtils.java | 4 ++-- .../server/utils/CalculatedFieldUtils.java | 10 ++++----- ...titiesAggregationCalculatedFieldTest.java} | 18 +++++++-------- .../common/data/cf/CalculatedFieldType.java | 2 +- .../CalculatedFieldConfiguration.java | 4 ++-- ...regationCalculatedFieldConfiguration.java} | 4 ++-- .../CalculatedFieldDataValidator.java | 4 ++-- 14 files changed, 60 insertions(+), 60 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{LatestValuesAggregationCalculatedFieldState.java => RelaredEntitiesAggregationCalculatedFieldState.java} (94%) rename application/src/test/java/org/thingsboard/server/cf/{LatestValuesAggregationCalculatedFieldTest.java => RelatedEntitiesAggregationCalculatedFieldTest.java} (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/{LatestValuesAggregationCalculatedFieldConfiguration.java => RelatedEntitiesAggregationCalculatedFieldConfiguration.java} (91%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1252b8091b..5a1ada08c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -54,7 +54,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -254,8 +254,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); Map updatedArgs = state.update(fetchedArgs, ctx); - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + relatedEntitiesAggState.setLastMetricsEvalTs(-1); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -271,7 +271,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { cleanupAggregationState(msg.getRelatedEntityId(), aggState); processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); } else { @@ -279,7 +279,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { state.getArguments().values().forEach(argEntry -> { AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; aggEntry.getAggInputs().remove(relatedEntityId); @@ -691,7 +691,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { fetchedArgs = fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 24fd5d320b..5b4528370b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -346,7 +346,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List matchingCfs = cfsByEntityIdAndProfile.stream() .filter(cf -> { - var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + var config = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); RelationPathLevel relation = config.getRelation(); return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); }) @@ -377,7 +377,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. @@ -411,7 +411,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (newCf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); @@ -518,7 +518,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + if (cf.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; @@ -753,7 +753,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 1c606cf840..3ddc8e1067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -98,7 +98,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -190,7 +190,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -202,7 +202,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); Map> argsFutures = aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 918d71e326..1c755ee05a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -83,7 +83,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cf); } } @@ -200,7 +200,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (configuration instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 147882c3c2..84f49c7c9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -190,7 +190,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cfCtx.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 89ed4e4401..46442847a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -139,7 +139,7 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) { relatedEntityArguments.put(refKey, entry.getKey()); continue; } @@ -185,7 +185,7 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; @@ -228,8 +228,8 @@ public class CalculatedFieldCtx { } initialized = true; } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + case RELATED_ENTITIES_AGGREGATION -> { + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); configuration.getMetrics().forEach((key, metric) -> { if (metric.getInput() instanceof AggFunctionInput functionInput) { initTbelExpression(functionInput.getFunction()); @@ -594,8 +594,8 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } @@ -617,7 +617,7 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } - if (hasLatestValuesAggregationConfigurationChanges(other)) { + if (hasRelatedEntitiesAggregationConfigurationChanges(other)) { return true; } return false; @@ -631,9 +631,9 @@ public class CalculatedFieldCtx { return false; } - private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java index fdc045a0db..b53042b1fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; @@ -46,7 +46,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -57,14 +57,14 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF private final Map> inputs = new HashMap<>(); - public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @Override public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { super.setCtx(ctx, actorCtx); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @@ -86,7 +86,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index d30b4eaf93..e1763ffa2d 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 6e65b4d5d9..67af13fdb3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -49,7 +49,7 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -240,8 +240,8 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + case RELATED_ENTITIES_AGGREGATION -> { + RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java similarity index 97% rename from application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java rename to application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java index eb11a329be..068c5f851d 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java @@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; @@ -66,7 +66,7 @@ import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTE @Slf4j @DaoSqlTest -public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { +public class RelatedEntitiesAggregationCalculatedFieldTest extends AbstractControllerTest { private Tenant savedTenant; @@ -470,7 +470,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Has"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); @@ -493,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); argument.setDefaultValue("false"); @@ -523,7 +523,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); AggMetric aggMetric = new AggMetric(); aggMetric.setInput(new AggKeyInput("temp")); aggMetric.setFilter("return temp < 100;"); @@ -559,7 +559,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); @@ -588,7 +588,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); @@ -730,9 +730,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll CalculatedField calculatedField = new CalculatedField(); calculatedField.setName(name); calculatedField.setEntityId(entityId); - calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + calculatedField.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); - LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); configuration.setDeduplicationIntervalInSec(deduplicationInterval); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index fe9ee7f9aa..4463c835db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -26,7 +26,7 @@ public enum CalculatedFieldType { GEOFENCING, ALARM, PROPAGATION, - LATEST_VALUES_AGGREGATION; + RELATED_ENTITIES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 3072b7c546..ca0a3c1a54 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -42,7 +42,7 @@ import java.util.stream.Collectors; @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), @Type(value = PropagationCalculatedFieldConfiguration.class, name = "PROPAGATION"), - @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") + @Type(value = RelatedEntitiesAggregationCalculatedFieldConfiguration.class, name = "RELATED_ENTITIES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java similarity index 91% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 5f8fb65265..69e4ee7fdf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { +public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { private RelationPathLevel relation; private Map arguments; @@ -36,7 +36,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 319a4fbe8b..3ccb837b3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -113,7 +113,7 @@ public class CalculatedFieldDataValidator extends DataValidator } private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { - if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + if (!(calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfiguration)) { return; } long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); From 2778f79e5e8a1d9262450a33edcfbc77d64e88c2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 12:45:29 +0300 Subject: [PATCH 065/122] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 80 +++++++------------ ...alculatedFieldManagerMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 14 ---- .../cf/CalculatedFieldProcessingService.java | 2 - ...faultCalculatedFieldProcessingService.java | 5 -- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../AggSingleEntityArgumentEntry.java | 1 - ...itiesAggregationCalculatedFieldState.java} | 64 ++++++++------- ...java => RelatedEntitiesArgumentEntry.java} | 20 ++--- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 18 ++--- ... => RelatedEntitiesArgumentEntryTest.java} | 27 ++----- 13 files changed, 96 insertions(+), 151 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{RelaredEntitiesAggregationCalculatedFieldState.java => RelatedEntitiesAggregationCalculatedFieldState.java} (84%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggArgumentEntry.java => RelatedEntitiesArgumentEntry.java} (72%) rename application/src/test/java/org/thingsboard/server/service/cf/ctx/state/{AggArgumentEntryTest.java => RelatedEntitiesArgumentEntryTest.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5a1ada08c5..434f555325 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,9 +52,8 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -227,17 +226,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); var state = states.get(ctx.getCfId()); try { - boolean justRestored = false; + Map updatedArgs = new HashMap<>(); if (state == null) { state = createState(ctx); - justRestored = true; + } else { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); + updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); } if (state.isSizeOk()) { - Map updatedArgs = new HashMap<>(); - if (!justRestored) { - updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); - } - processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); + processStateIfReady(state, updatedArgs, ctx, Collections.singletonList(ctx.getCfId()), null, null, callback); } else { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } @@ -250,19 +251,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { - Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); - Map updatedArgs = state.update(fetchedArgs, ctx); - - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { - relatedEntitiesAggState.setLastMetricsEvalTs(-1); - } - - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); - - return updatedArgs; - } - private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { CalculatedFieldCtx ctx = msg.getCalculatedField(); CalculatedFieldId cfId = ctx.getCfId(); @@ -271,34 +259,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { - cleanupAggregationState(msg.getRelatedEntityId(), aggState); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { + aggState.cleanupEntityData(msg.getRelatedEntityId()); + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + if (state.isSizeOk()) { + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } } else { + // todo: log msg.getCallback().onSuccess(); } } - private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { - state.getArguments().values().forEach(argEntry -> { - AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); - }); - state.getInputs().remove(relatedEntityId); - state.setLastMetricsEvalTs(-1); - } - - @SneakyThrows - private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - return argumentsFuture.get(1, TimeUnit.MINUTES); - } - - public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -692,17 +668,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = fetchedArgs.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) - )); + fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } + private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + return fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + )); + } + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { List cfIds = new LinkedList<>(); for (var cfId : proto.getPreviousCalculatedFieldsList()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 5b4528370b..7f1c7b1925 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -469,8 +469,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); log.debug("Received telemetry msg from entity [{}]", entityId); - // 3 = 1 for CF processing + 1 for links processing + 1 for owner entity processing - MultipleTbCallback callback = new MultipleTbCallback(3, msg.getCallback()); + // 4 = 1 for CF processing + 1 for links processing + 1 for owner entity processing + 1 for aggregation processing + MultipleTbCallback callback = new MultipleTbCallback(4, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 3ddc8e1067..0c0f2f23e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -201,20 +201,6 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } - protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - Map> argsFutures = aggConfig.getArguments().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) - )); - - return Futures.whenAllComplete(argsFutures.values()) - .call(() -> resolveArgumentFutures(argsFutures), - MoreExecutors.directExecutor()); - } - private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 52b3341151..a9139572b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,8 +33,6 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); - Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9074c1036f..d7957dce9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,11 +91,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } - @Override - public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); - } - @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index c8f7dd0c3d..5bb16292be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,7 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -77,7 +77,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { - return new AggArgumentEntry(entityIdkvEntryMap, false); + return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 9882b8181b..5c672cf04e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6430fe3f1c..b935256860 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; - private boolean deleted; public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index b53042b1fe..c0baca836c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -46,7 +45,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -55,9 +54,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat private long deduplicationInterval = -1; private Map metrics; - private final Map> inputs = new HashMap<>(); - - public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { + public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @@ -75,7 +72,6 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; - inputs.clear(); } @Override @@ -91,17 +87,8 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public Map update(Map argumentValues, CalculatedFieldCtx ctx) { - Map updatedArguments = super.update(argumentValues, ctx); lastArgsRefreshTs = System.currentTimeMillis(); - for (Map.Entry argEntry : arguments.entrySet()) { - String key = argEntry.getKey(); - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); - Map aggInputs = aggArgumentEntry.getAggInputs(); - aggInputs.forEach((entityId, argumentEntry) -> { - inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); - }); - } - return updatedArguments; + return super.update(argumentValues, ctx); } @Override @@ -115,7 +102,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() @@ -124,20 +111,47 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } + public Map updateEntityData(Map fetchedArgs) { + lastMetricsEvalTs = -1; + return update(fetchedArgs, ctx); + } + + public void cleanupEntityData(EntityId relatedEntityId) { + arguments.values().forEach(argEntry -> { + RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + lastMetricsEvalTs = -1; + lastArgsRefreshTs = System.currentTimeMillis(); + } + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } + private Map> prepareInputs() { + Map> inputs = new HashMap<>(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); + relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return inputs; + } + private ObjectNode aggregateMetrics(Output output) throws Exception { ObjectNode aggResult = JacksonUtil.newObjectNode(); + Map> inputs = prepareInputs(); for (Entry entry : metrics.entrySet()) { String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); - aggregateMetric(metric, aggMetricEntry); + aggregateMetric(metric, aggMetricEntry, inputs); aggMetricEntry.result().ifPresent(result -> { aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); }); @@ -145,7 +159,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return aggResult; } - private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + private void aggregateMetric(AggMetric metric, AggEntry aggEntry, Map> inputs) throws Exception { for (Map entityInputs : inputs.values()) { if (applyAggregation(metric.getFilter(), entityInputs)) { Object arg = resolveAggregationInput(metric.getInput(), entityInputs); @@ -183,16 +197,4 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { - long latestTs = getLatestTimestamp(); - if (useLatestTs && latestTs != -1) { - ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", latestTs); - resultNode.set("values", result); - return resultNode; - } else { - return result; - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index e002aece2c..5385808923 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -27,7 +27,7 @@ import java.util.Map; @Data @AllArgsConstructor -public class AggArgumentEntry implements ArgumentEntry { +public class RelatedEntitiesArgumentEntry implements ArgumentEntry { private final Map aggInputs; @@ -35,7 +35,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST; + return ArgumentEntryType.RELATED_ENTITIES; } @Override @@ -45,19 +45,15 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggArgumentEntry aggArgumentEntry) { - aggInputs.putAll(aggArgumentEntry.aggInputs); + if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - if (aggSingleEntityArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); } else { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); - if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); - } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); - } + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index e1763ffa2d..239ffedbc7 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 67af13fdb3..389fad8ff3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,9 +47,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -104,9 +104,9 @@ public class CalculatedFieldUtils { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); - case AGGREGATE_LATEST -> { - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry; - aggArgumentEntry.getAggInputs() + case RELATED_ENTITIES -> { + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; + relatedEntitiesArgumentEntry.getAggInputs() .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } } @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -241,7 +241,7 @@ public class CalculatedFieldUtils { } } case RELATED_ENTITIES_AGGREGATION -> { - RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; + RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { @@ -249,7 +249,7 @@ public class CalculatedFieldUtils { arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); }); aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java similarity index 80% rename from application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java rename to application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index c1c2d85c55..14d2801e0e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; @@ -31,9 +31,9 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -public class AggArgumentEntryTest { +public class RelatedEntitiesArgumentEntryTest { - private AggArgumentEntry entry; + private RelatedEntitiesArgumentEntry entry; private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); @@ -46,7 +46,7 @@ public class AggArgumentEntryTest { aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); - entry = new AggArgumentEntry(aggInputs, false); + entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @Test @@ -61,17 +61,17 @@ public class AggArgumentEntryTest { DeviceId device3 = new DeviceId(UUID.randomUUID()); DeviceId device4 = new DeviceId(UUID.randomUUID()); - AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); - assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); Map aggInputs = entry.getAggInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); } @Test @@ -98,15 +98,4 @@ public class AggArgumentEntryTest { assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } - @Test - void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - - Map aggInputs = entry.getAggInputs(); - assertThat(aggInputs.size()).isEqualTo(1); - assertThat(aggInputs.get(device2)).isNull(); - } - } From 7bf7e0f994e4d69c5ca2f1f8e7464021c9310f48 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 15:33:02 +0300 Subject: [PATCH 066/122] removed unnecessary fetch arguments methods --- ...tractCalculatedFieldProcessingService.java | 77 ++++++------------- ...faultCalculatedFieldProcessingService.java | 3 - .../service/cf/ctx/state/ArgumentEntry.java | 8 +- .../utils/CalculatedFieldArgumentUtils.java | 9 --- 4 files changed, 27 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 0c0f2f23e9..cfd7c9af3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -64,7 +64,6 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; -import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -98,7 +97,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchRelatedEntitiesAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -128,23 +127,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { - ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - - return Futures.transform(relationsFut, relations -> { - if (relations == null) { - return new ArrayList<>(); - } - - return switch (relation.direction()) { - case FROM -> relations.stream() - .map(EntityRelation::getTo) - .toList(); - case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); - }; - }, calculatedFieldCallbackExecutor); - } - protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -189,7 +171,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } - protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + protected Map> fetchRelatedEntitiesAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -197,10 +179,27 @@ public abstract class AbstractCalculatedFieldProcessingService { return aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchRelatedEntitiesArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) )); } + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); + + return Futures.transform(relationsFut, relations -> { + if (relations == null) { + return new ArrayList<>(); + } + + return switch (relation.direction()) { + case FROM -> relations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -252,11 +251,11 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + public ListenableFuture fetchRelatedEntitiesArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { List>> futures = aggEntities.stream() .map(entityId -> { - ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); - return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); @@ -321,34 +320,4 @@ public abstract class AbstractCalculatedFieldProcessingService { return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); } - private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { - return switch (argument.getRefEntityKey().getType()) { - case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); - case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); - case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); - }; - } - - private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { - log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); - var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); - return Futures.transform(attributeOptFuture, attrOpt -> { - log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); - return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { - String key = argument.getRefEntityKey().getKey(); - log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); - return Futures.transform( - timeseriesService.findLatest(tenantId, entityId, key), - result -> { - log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); - return transformAggSingleArgument(entityId, tsKvEntry); - }, calculatedFieldCallbackExecutor); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index d7957dce9b..52393d0ffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -56,7 +54,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 5bb16292be..863d6f5b50 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -80,8 +80,8 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleEntityArgumentEntry(entityId, kvEntry); + static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new AggSingleEntityArgumentEntry(entityId, argumentEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 239ffedbc7..0c0d401688 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -56,14 +55,6 @@ public class CalculatedFieldArgumentUtils { } } - public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); - } else { - return new AggSingleEntityArgumentEntry(); - } - } - public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); From 899dd9002bb045042ca439566eefa4efe55bd4d3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 17:00:55 +0300 Subject: [PATCH 067/122] removed single agg argument and updated old single value argument --- ...CalculatedFieldEntityMessageProcessor.java | 15 ++- ...tractCalculatedFieldProcessingService.java | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 12 +-- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 15 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 30 ++++++ .../AggSingleEntityArgumentEntry.java | 97 ----------------- ...titiesAggregationCalculatedFieldState.java | 16 +-- .../RelatedEntitiesArgumentEntry.java | 13 +-- .../state/aggregation/function/AggEntry.java | 15 ++- .../function/AggFunctionFactory.java | 33 ------ .../aggregation/function/AvgAggEntry.java | 6 +- .../aggregation/function/BaseAggEntry.java | 9 +- .../aggregation/function/CountAggEntry.java | 2 +- .../function/CountUniqueAggEntry.java | 2 +- .../aggregation/function/MaxAggEntry.java | 5 +- .../aggregation/function/MinAggEntry.java | 5 +- .../aggregation/function/SumAggEntry.java | 5 +- .../server/utils/CalculatedFieldUtils.java | 61 ++++------- .../AggSingleEntityArgumentEntryTest.java | 101 ------------------ .../RelatedEntitiesArgumentEntryTest.java | 17 ++- .../configuration/aggregation/AggInput.java | 2 + ...gregationCalculatedFieldConfiguration.java | 12 +-- common/proto/src/main/proto/queue.proto | 14 +-- .../thingsboard/script/api/tbel/TbUtils.java | 11 ++ .../script/api/tbel/TbelCfArg.java | 2 +- ... => TbelCfRelatedEntitiesAggregation.java} | 4 +- 30 files changed, 148 insertions(+), 365 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfLatestValuesAggregation.java => TbelCfRelatedEntitiesAggregation.java} (90%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 434f555325..ca97dae51b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,7 +52,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -232,7 +231,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); - updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + updatedArgs = relatedEntitiesAggState.updateEntityData(setEntityIdToSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -544,7 +543,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); + arguments.put(argName, new SingleValueArgumentEntry(originator, item)); } } } @@ -597,7 +596,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); + arguments.put(argName, new SingleValueArgumentEntry(entityId, item)); } } } @@ -636,7 +635,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + arguments.put(argName, new SingleValueArgumentEntry(msgEntityId, argumentEntry)); } } } @@ -668,18 +667,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); + fetchedArgs = setEntityIdToSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } - private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + private Map setEntityIdToSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { return fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + argEntry -> new SingleValueArgumentEntry(relatedEntityId, argEntry.getValue()) )); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index cfd7c9af3e..d4bf0d52be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -255,7 +255,7 @@ public abstract class AbstractCalculatedFieldProcessingService { List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); - return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createSingleValueArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 863d6f5b50..b331c11a47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,6 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,8 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), - @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES") }) public interface ArgumentEntry { @@ -64,6 +62,10 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } + static ArgumentEntry createSingleValueArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new SingleValueArgumentEntry(entityId, argumentEntry); + } + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); } @@ -80,8 +82,4 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { - return new AggSingleEntityArgumentEntry(entityId, argumentEntry); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 5c672cf04e..427df2bf5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 91c331f88e..d48ed9c268 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -18,13 +18,12 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -76,7 +75,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; @@ -152,14 +151,4 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } - protected Object formatResult(double result, Integer decimals) { - if (decimals == null) { - return result; - } - if (decimals.equals(0)) { - return TbUtils.toInt(result); - } - return TbUtils.toFixed(result, decimals); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 46442847a0..aab858d85d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -634,7 +634,7 @@ public class CalculatedFieldCtx { private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); + return !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 8e202308d0..2b3ba19528 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalcul import java.io.Closeable; import java.util.Map; -import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index a268577d4b..ab0ed26dfe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -52,7 +52,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = ctx.evaluateSimpleExpression(expression.get(), this); Output output = ctx.getOutput(); - Object result = formatResult(expressionResult, output.getDecimalsByDefault()); + Object result = TbUtils.roundResult(expressionResult, output.getDecimalsByDefault()); JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e81201c961..67fb385917 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.springframework.lang.Nullable; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -37,6 +39,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + @Nullable + protected EntityId entityId; + protected long ts; protected BasicKvEntry kvEntryValue; protected Long version; @@ -45,6 +50,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(EntityId entityId, ArgumentEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { this.ts = singleValueArgumentEntry.ts; @@ -54,6 +64,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } } + public SingleValueArgumentEntry(EntityId entityId, TsKvProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { @@ -62,6 +77,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } + public SingleValueArgumentEntry(EntityId entityId, AttributeValueProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); if (entry.hasVersion()) { @@ -70,6 +90,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } + public SingleValueArgumentEntry(EntityId entityId, KvEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); @@ -81,6 +106,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } + public SingleValueArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + this(ts, kvEntryValue, version); + this.entityId = entityId; + } + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { this.ts = ts; this.kvEntryValue = kvEntryValue; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java deleted file mode 100644 index b935256860..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; -import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { - - private EntityId entityId; - - public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { - super(ts, kvEntryValue, version); - this.entityId = entityId; - } - - @Override - public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { - if (aggSingleEntityEntry.isForceResetPrevious()) { - return applyNewEntry(aggSingleEntityEntry); - } - - if (aggSingleEntityEntry.getTs() < this.ts) { - if (!isDefaultValue()) { - return false; - } - } - - Long newVersion = aggSingleEntityEntry.getVersion(); - if (newVersion == null || this.version == null || newVersion > this.version) { - return applyNewEntry(aggSingleEntityEntry); - } - } else { - throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); - } - return false; - } - - private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { - this.ts = entry.getTs(); - this.version = entry.getVersion(); - this.kvEntryValue = entry.getKvEntryValue(); - this.entityId = entry.getEntityId(); - return true; - } - - @Override - public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c0baca836c..8e78824c7c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -37,7 +37,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; import java.util.HashMap; import java.util.Map; @@ -150,10 +149,10 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); - AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + AggEntry aggMetricEntry = AggEntry.createAggFunction(metric.getFunction()); aggregateMetric(metric, aggMetricEntry, inputs); - aggMetricEntry.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + aggMetricEntry.result(output.getDecimalsByDefault()).ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); }); } return aggResult; @@ -188,13 +187,4 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - private Object formatResult(Object aggregationResult, Integer decimals) { - try { - double result = Double.parseDouble(aggregationResult.toString()); - return formatResult(result, decimals); - } catch (Exception e) { - throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5385808923..45b7755af4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,10 +18,11 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; @@ -48,12 +49,12 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,7 +69,7 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return new TbelCfLatestValuesAggregation(aggInputs.values()); + return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java index 4239e94fec..c4b93fd91d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; @@ -36,10 +37,22 @@ import java.util.Optional; }) public interface AggEntry { + @JsonIgnore AggFunction getType(); void update(Object value); - Optional result(); + Optional result(Integer precision); + + static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java deleted file mode 100644 index 5ccc355b1f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation.function; - -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; - -public class AggFunctionFactory { - - public static AggEntry createAggFunction(AggFunction function) { - return switch (function) { - case MIN -> new MinAggEntry(); - case MAX -> new MaxAggEntry(); - case SUM -> new SumAggEntry(); - case AVG -> new AvgAggEntry(); - case COUNT -> new CountAggEntry(); - case COUNT_UNIQUE -> new CountUniqueAggEntry(); - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index afe6abb93e..e063ff2ea2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -34,8 +35,9 @@ public class AvgAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); + protected Object prepareResult(Integer precision) { + double result = sum.divide(BigDecimal.valueOf(count), RoundingMode.HALF_UP).doubleValue(); + return TbUtils.roundResult(result, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index 0c12bd13c0..b320435e99 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -28,10 +28,10 @@ public abstract class BaseAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { if (hasResult) { hasResult = false; - return Optional.of(prepareResult()); + return Optional.of(prepareResult(precision)); } else { return Optional.empty(); } @@ -39,10 +39,13 @@ public abstract class BaseAggEntry implements AggEntry { protected abstract void doUpdate(double value); - protected abstract double prepareResult(); + protected abstract Object prepareResult(Integer precision); protected double extractDoubleValue(Object value) { try { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } return Double.parseDouble(value.toString()); } catch (Exception e) { throw new NumberFormatException("Cannot parse value " + value.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java index 469048d62d..09116985d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -29,7 +29,7 @@ public class CountAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(count); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index b8b0b92470..efb4a58c90 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -34,7 +34,7 @@ public class CountUniqueAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(items.size()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index 6e4235b72f..ddc47daf33 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MaxAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MaxAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return max; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(max, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java index eeacc3a7d9..e517ad305f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MinAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MinAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return min; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(min, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java index b90817d784..fe29d27b7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -31,8 +32,8 @@ public class SumAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.doubleValue(); + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(sum.doubleValue(), precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 389fad8ff3..d75ec8a70a 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -35,7 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -47,9 +46,8 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -98,7 +96,7 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); + RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); @@ -107,7 +105,7 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -122,7 +120,7 @@ public class CalculatedFieldUtils { } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setLatestValuesAggregationState(aggBuilder.build()); + builder.setRelatedEntitiesAggregationState(aggBuilder.build()); } return builder.build(); } @@ -145,17 +143,6 @@ public class CalculatedFieldUtils { return ruleState; } - public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { - AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() - .setEntityId(ProtoUtils.toProto(entityId)); - - if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } - - return builder.build(); - } - public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -166,6 +153,10 @@ public class CalculatedFieldUtils { Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + if (entry.getEntityId() != null) { + builder.setEntityId(ProtoUtils.toProto(entry.getEntityId())); + } + return builder.build(); } @@ -242,11 +233,11 @@ public class CalculatedFieldUtils { } case RELATED_ENTITIES_AGGREGATION -> { RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); + RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); @@ -258,31 +249,19 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { - if (!proto.hasValue()) { - return new AggSingleEntityArgumentEntry(); - } - EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); - SingleValueArgumentProto singleValueArgument = proto.getValue(); - TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleEntityArgumentEntry( - entityId, - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), - singleValueArgument.getVersion() - ); - } - public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); } TsValueProto tsValueProto = proto.getValue(); - return new SingleValueArgumentEntry( - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), - proto.getVersion() - ); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + long ts = tsValueProto.getTs(); + long version = proto.getVersion(); + if (proto.hasEntityId()) { + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + return new SingleValueArgumentEntry(entityId, ts, kvEntry, version); + } + return new SingleValueArgumentEntry(ts, kvEntry, version); } public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java deleted file mode 100644 index 0401bb0156..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class AggSingleEntityArgumentEntryTest { - - private AggSingleEntityArgumentEntry entry; - - private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); - - private final long ts = System.currentTimeMillis(); - - @BeforeEach - void setUp() { - entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); - } - - @Test - void testUpdateEntryWhenNotAggEntryPassed() { - assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); - } - - @Test - void testUpdateEntryWhenResetPrevious() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); - singleEntityArgumentEntry.setForceResetPrevious(true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); - assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); - assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); - } - - - @Test - void testUpdateEntryWithTheSameTsAndVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); - } - - @Test - void testUpdateEntryWithTheSameTsAndDifferentVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); - } - - @Test - void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(56L); - assertThat(entry.getVersion()).isNull(); - } - - @Test - void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(76L); - assertThat(entry.getVersion()).isEqualTo(23); - } - - @Test - void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); - } - - @Test - void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); - } - - @Test - void testUpdateEntryWithOldTs() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 14d2801e0e..61b45b83c9 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; import java.util.Map; @@ -43,8 +42,8 @@ public class RelatedEntitiesArgumentEntryTest { @BeforeEach void setUp() { Map aggInputs = new HashMap<>(); - aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); - aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + aggInputs.put(device1, new SingleValueArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @@ -62,8 +61,8 @@ public class RelatedEntitiesArgumentEntryTest { DeviceId device4 = new DeviceId(UUID.randomUUID()); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( - device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), - device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + device3, new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new SingleValueArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); @@ -75,10 +74,10 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndNoEntriesById() { DeviceId device3 = new DeviceId(UUID.randomUUID()); - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); @@ -88,8 +87,8 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndEntryByIdExist() { + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java index fac988d5d9..06929de81c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInput { + @JsonIgnore String getType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 69e4ee7fdf..9d4c7bdaf6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,9 +30,12 @@ import java.util.Map; @Data public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + @NotNull private RelationPathLevel relation; private Map arguments; private long deduplicationIntervalInSec; + @Valid + @NotEmpty private Map metrics; private Output output; private boolean useLatestTs; @@ -41,9 +47,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A @Override public void validate() { - if (relation == null) { - throw new IllegalArgumentException("Relation must be specified!"); - } relation.validate(); if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); @@ -51,9 +54,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); } - if (metrics.isEmpty()) { - throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); - } } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b59e01128b..2e3a2387e7 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -888,6 +888,7 @@ message SingleValueArgumentProto { string argName = 1; TsValueProto value = 2; int64 version = 3; + EntityIdProto entityId = 4; } message TsDoubleValProto { @@ -915,14 +916,9 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message AggSingleArgumentEntryProto { - EntityIdProto entityId = 1; - SingleValueArgumentProto value = 2; -} - -message LatestValuesAggregationStateProto { +message RelatedEntitiesAggregationStateProto { int64 lastArgsUpdateTs = 1; - repeated AggSingleArgumentEntryProto aggArguments = 2; + repeated SingleValueArgumentProto aggArguments = 2; } message CalculatedFieldStateProto { @@ -932,7 +928,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - LatestValuesAggregationStateProto latestValuesAggregationState = 7; + RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1303,7 +1299,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 15; + bool relationChanged = 14; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 072a17835d..3e677cf269 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1186,6 +1186,17 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); } + // todo: register method + public static Object roundResult(double value, Integer precision) { + if (precision == null) { + return value; + } + if (precision.equals(0)) { + return toInt(value); + } + return toFixed(value, precision); + } + public static boolean isNaN(double value) { return Double.isNaN(value); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 856cc4bd39..62d6d3d002 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java similarity index 90% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 4d1b42aa94..75c17f0a0e 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data -public class TbelCfLatestValuesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { private final Object value; @JsonCreator - public TbelCfLatestValuesAggregation( + public TbelCfRelatedEntitiesAggregation( @JsonProperty("value") Object value ) { this.value = value; From d97d932cfbe3bc25705f73b6a1f57af15d1b4419 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:10:08 +0300 Subject: [PATCH 068/122] refactoring --- ...alculatedFieldManagerMessageProcessor.java | 33 +++++++------- .../cf/TelemetryCalculatedFieldResult.java | 8 ++-- .../ctx/state/BaseCalculatedFieldState.java | 18 +++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 32 +++++--------- ...titiesAggregationCalculatedFieldState.java | 16 +++---- .../RelatedEntitiesArgumentEntry.java | 4 ++ .../aggregation/function/BaseAggEntry.java | 4 +- .../aggregation/function/MaxAggEntry.java | 2 +- .../queue/DefaultTbClusterService.java | 7 +-- .../server/utils/CalculatedFieldUtils.java | 43 ++++++++++--------- .../data/plugin/ComponentLifecycleEvent.java | 4 +- common/proto/src/main/proto/queue.proto | 9 ++-- .../script/api/tbel/TbelCfArg.java | 2 +- .../TbelCfRelatedEntitiesAggregation.java | 2 +- 14 files changed, 95 insertions(+), 89 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7f1c7b1925..3d61825ebe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -187,9 +187,18 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { + var event = msg.getData().getEvent(); + if (msg.getData().isRelationChanged()) { + log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); + switch (event) { + case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); + case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); + } + return; + } log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); - var event = msg.getData().getEvent(); switch (entityType) { case CALCULATED_FIELD -> { switch (event) { @@ -280,26 +289,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); - } else if (msg.isRelationChanged()) { - onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - if (msg.isRelationChanged()) { - onRelationDeleted(msg, callback); - } else { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); - } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); - } + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java index 1ad666eac5..d59ec9cca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java @@ -39,6 +39,8 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu private final AttributeScope scope; private final JsonNode result; + public static TelemetryCalculatedFieldResult EMPTY = TelemetryCalculatedFieldResult.builder().result(null).build(); + @Override public TbMsg toTbMsg(EntityId entityId, List cfIds) { TbMsgType msgType = switch (type) { @@ -66,9 +68,9 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu @Override public boolean isEmpty() { return result == null || result.isMissingNode() || result.isNull() || - (result.isObject() && result.isEmpty()) || - (result.isArray() && result.isEmpty()) || - (result.isTextual() && result.asText().isEmpty()); + (result.isObject() && result.isEmpty()) || + (result.isArray() && result.isEmpty()) || + (result.isTextual() && result.asText().isEmpty()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d48ed9c268..d8f13e6a20 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -20,10 +20,10 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -75,9 +75,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); - arguments.put(key, newEntry); + if (existingEntry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + relatedEntitiesArgumentEntry.updateEntry(newEntry); + } else { + arguments.put(key, newEntry); + } entryUpdated = true; } else { entryUpdated = existingEntry.updateEntry(newEntry); @@ -110,7 +114,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, @Override public boolean isReady() { return arguments.keySet().containsAll(requiredArguments) && - arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); } @Override @@ -122,9 +126,11 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, } @Override - public void close() {} + public void close() { + } - protected void validateNewEntry(String key, ArgumentEntry newEntry) {} + protected void validateNewEntry(String key, ArgumentEntry newEntry) { + } protected ObjectNode toSimpleResult(boolean useLatestTs, ObjectNode valuesNode) { if (!useLatestTs) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index aab858d85d..40e414920a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -263,33 +263,23 @@ public class CalculatedFieldCtx { } public ListenableFuture evaluateTbelExpression(String expression, CalculatedFieldState state) { - return evaluateTbelExpression(tbelExpressions.get(expression), state); + return evaluateTbelExpression(tbelExpressions.get(expression), state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, CalculatedFieldState state) { - Map arguments = new LinkedHashMap<>(); - List args = new ArrayList<>(argNames.size() + 1); - args.add(new Object()); // first element is a ctx, but we will set it later; - for (String argName : argNames) { - var arg = toTbelArgument(argName, state); - arguments.put(argName, arg); - if (arg instanceof TbelCfSingleValueArg svArg) { - args.add(svArg.getValue()); - } else { - args.add(arg); - } - } - args.set(0, new TbelCfCtx(arguments, state.getLatestTimestamp())); - - return expression.executeScriptAsync(args.toArray()); + return evaluateTbelExpression(expression, state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + return evaluateTbelExpression(tbelExpressions.get(expression), entries, latestTimestamp); + } + + public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, Map entries, long latestTimestamp) { Map arguments = new LinkedHashMap<>(); List args = new ArrayList<>(argNames.size() + 1); args.add(new Object()); // first element is a ctx, but we will set it later; for (String argName : argNames) { - var arg = entries.get(argName).toTbelCfArg(); + var arg = toTbelArgument(argName, entries); arguments.put(argName, arg); if (arg instanceof TbelCfSingleValueArg svArg) { args.add(svArg.getValue()); @@ -299,7 +289,7 @@ public class CalculatedFieldCtx { } args.set(0, new TbelCfCtx(arguments, latestTimestamp)); - return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + return expression.executeScriptAsync(args.toArray()); } public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { @@ -308,8 +298,8 @@ public class CalculatedFieldCtx { return systemContext.scheduleMsgWithDelay(actorCtx, new CalculatedFieldReevaluateMsg(tenantId, this), delayMs); } - private TbelCfArg toTbelArgument(String key, CalculatedFieldState state) { - return state.getArguments().get(key).toTbelCfArg(); + private TbelCfArg toTbelArgument(String key, Map arguments) { + return arguments.get(key).toTbelCfArg(); } private void initTbelExpression(String expression) { @@ -658,7 +648,7 @@ public class CalculatedFieldCtx { yield true; } yield geofencingState.getLastDynamicArgumentsRefreshTs() < - System.currentTimeMillis() - scheduledUpdateIntervalMillis; + System.currentTimeMillis() - scheduledUpdateIntervalMillis; } default -> false; }; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 8e78824c7c..c8731b71f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -42,6 +42,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import static java.util.concurrent.TimeUnit.SECONDS; + @Slf4j @Getter public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @@ -50,7 +52,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; - private long deduplicationInterval = -1; + private long deduplicationIntervalMs = -1; private Map metrics; public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { @@ -62,7 +64,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat super.setCtx(ctx, actorCtx); var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalInSec(); + deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); } @Override @@ -76,7 +78,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public void init() { super.init(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); } @Override @@ -97,16 +99,14 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat Output output = ctx.getOutput(); ObjectNode aggResult = aggregateMetrics(output); lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); } } @@ -125,7 +125,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } private boolean shouldRecalculate() { - boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationIntervalMs; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 45b7755af4..5b97b1ed0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -50,6 +50,10 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (entry.isForceResetPrevious()) { + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + return true; + } ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index b320435e99..8ca523938d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -43,8 +43,8 @@ public abstract class BaseAggEntry implements AggEntry { protected double extractDoubleValue(Object value) { try { - if (value instanceof Number) { - return ((Number) value).doubleValue(); + if (value instanceof Number number) { + return number.doubleValue(); } return Double.parseDouble(value.toString()); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index ddc47daf33..6d734a5a08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti public class MaxAggEntry extends BaseAggEntry { - private double max = -Double.MAX_VALUE; + private double max = Double.MIN_VALUE; @Override protected void doUpdate(double value) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 22faeaf61b..5b6596c9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -741,7 +741,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.UPDATED) + .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -753,7 +753,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.DELETED) + .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -809,7 +809,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d75ec8a70a..ba3aa3fd53 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,16 +95,18 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { - case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); - case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); - case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case SINGLE_VALUE -> + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> + builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> + builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); + .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -119,8 +120,7 @@ public class CalculatedFieldUtils { } } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { - aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setRelatedEntitiesAggregationState(aggBuilder.build()); + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); } return builder.build(); } @@ -208,6 +208,20 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map> arguments = new HashMap<>(); + proto.getSingleValueArgumentsList().forEach(argProto -> { + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + relatedEntitiesAggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); + }); + relatedEntitiesAggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + + return relatedEntitiesAggState; + } + proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); @@ -231,19 +245,6 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case RELATED_ENTITIES_AGGREGATION -> { - RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); - Map> arguments = new HashMap<>(); - aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); - }); - arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); - }); - aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); - } } return state; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index 5d13db2348..31cab71e0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -32,7 +32,9 @@ public enum ComponentLifecycleEvent implements Serializable { STOPPED(5), DELETED(6), FAILED(7), - DEACTIVATED(8); + DEACTIVATED(8), + RELATION_UPDATED(9), + RELATION_DELETED(10); @Getter private final int protoNumber; // corresponds to ComponentLifecycleEvent proto diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 2e3a2387e7..bd441dd679 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -916,11 +916,6 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message RelatedEntitiesAggregationStateProto { - int64 lastArgsUpdateTs = 1; - repeated SingleValueArgumentProto aggArguments = 2; -} - message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -928,7 +923,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; + int64 lastArgsUpdateTs = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1281,6 +1276,8 @@ enum ComponentLifecycleEvent { DELETED = 6; FAILED = 7; DEACTIVATED = 8; + RELATION_UPDATED = 9; + RELATION_DELETED = 10; } message ComponentLifecycleMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 62d6d3d002..2fb12917ff 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 75c17f0a0e..3373aa2474 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -34,7 +34,7 @@ public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { @Override public String getType() { - return "LATEST_VALUES_AGGREGATION"; + return "RELATED_ENTITIES_AGGREGATION"; } @Override From c46d2f041568d6a7bb2e851d2bd00d192744d576 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:49:07 +0300 Subject: [PATCH 069/122] added related entities argument entry implementation --- ...titiesAggregationCalculatedFieldState.java | 4 +-- .../RelatedEntitiesArgumentEntry.java | 25 ++++++++++++------- .../function/CountUniqueAggEntry.java | 1 - .../server/utils/CalculatedFieldUtils.java | 2 +- .../RelatedEntitiesArgumentEntryTest.java | 10 ++++---- .../script/api/tbel/TbelCfArg.java | 2 +- ...> TbelCfRelatedEntitiesArgumentValue.java} | 17 +++++++------ 7 files changed, 34 insertions(+), 27 deletions(-) rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfRelatedEntitiesAggregation.java => TbelCfRelatedEntitiesArgumentValue.java} (68%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c8731b71f7..7e530b6809 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -118,7 +118,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat public void cleanupEntityData(EntityId relatedEntityId) { arguments.values().forEach(argEntry -> { RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); + aggEntry.getEntityInputs().remove(relatedEntityId); }); lastMetricsEvalTs = -1; lastArgsRefreshTs = System.currentTimeMillis(); @@ -135,7 +135,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat for (Map.Entry argEntry : arguments.entrySet()) { String key = argEntry.getKey(); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); - relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + relatedEntitiesArgumentEntry.getEntityInputs().forEach((entityId, argumentEntry) -> { inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5b97b1ed0a..2abe78d243 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,19 +18,21 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesArgumentValue; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; +import java.util.stream.Collectors; @Data @AllArgsConstructor public class RelatedEntitiesArgumentEntry implements ArgumentEntry { - private final Map aggInputs; + private final Map entityInputs; private boolean forceResetPrevious; @@ -41,24 +43,24 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public Object getValue() { - return aggInputs; + return entityInputs; } @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { - aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); + entityInputs.putAll(relatedEntitiesArgumentEntry.entityInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (entry.isForceResetPrevious()) { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); return true; } - ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = entityInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,12 +70,17 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public boolean isEmpty() { - return aggInputs.isEmpty(); + return entityInputs.isEmpty(); } @Override public TbelCfArg toTbelCfArg() { - return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); + var inputs = entityInputs.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey().getId(), + e -> (TbelCfSingleValueArg) e.getValue().toTbelCfArg() + )); + return new TbelCfRelatedEntitiesArgumentValue(inputs); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index efb4a58c90..a66cbaa6af 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; -import java.util.HashSet; import java.util.Optional; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index ba3aa3fd53..fd16245695 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -105,7 +105,7 @@ public class CalculatedFieldUtils { builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; - relatedEntitiesArgumentEntry.getAggInputs() + relatedEntitiesArgumentEntry.getEntityInputs() .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 61b45b83c9..cc60b249ac 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -67,10 +67,10 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device4)); } @Test @@ -81,7 +81,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(3); assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); } @@ -92,7 +92,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(2); assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 2fb12917ff..4f2719fb75 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesArgumentValue.class, name = "RELATED_ENTITIES_ARGUMENT_VALUE") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java similarity index 68% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java index 3373aa2474..02d641d576 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java @@ -19,22 +19,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + @Data -public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesArgumentValue implements TbelCfArg { - private final Object value; + private final Map entityInputs; @JsonCreator - public TbelCfRelatedEntitiesAggregation( - @JsonProperty("value") Object value - ) { - this.value = value; + public TbelCfRelatedEntitiesArgumentValue(@JsonProperty("entityInputs") Map values) { + this.entityInputs = Collections.unmodifiableMap(values); } - @Override public String getType() { - return "RELATED_ENTITIES_AGGREGATION"; + return "RELATED_ENTITIES_ARGUMENT_VALUE"; } @Override From cd15206061d80f9998d95273613ffa3f9126347d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 12:12:09 +0300 Subject: [PATCH 070/122] relation changed processing --- ...alculatedFieldManagerMessageProcessor.java | 57 ++++++++----------- .../queue/DefaultTbClusterService.java | 2 - .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 - common/proto/src/main/proto/queue.proto | 1 - 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3d61825ebe..96f7e9aa80 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -77,6 +78,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import java.util.function.Function; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -188,16 +190,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { var event = msg.getData().getEvent(); - if (msg.getData().isRelationChanged()) { - log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); - switch (event) { - case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); - case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); - default -> msg.getCallback().onSuccess(); - } + if (ComponentLifecycleEvent.RELATION_UPDATED.equals(event) || ComponentLifecycleEvent.RELATION_DELETED.equals(event)) { + log.debug("Processing relation [{}] event from entity: [{}]", event, msg.getData().getEntityId()); + onRelationChangedEvent(msg.getData(), msg.getCallback()); return; } - log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", event, msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); switch (entityType) { case CALCULATED_FIELD -> { @@ -306,36 +304,29 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } - private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { - callback.onSuccess(); - } - } + private void onRelationChangedEvent(ComponentLifecycleMsg msg, TbCallback callback) { + Function> relationAction = switch (msg.getEvent()) { + case RELATION_UPDATED -> relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_DELETED -> relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); + default -> null; + }; - private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { + if (relationAction == null) { callback.onSuccess(); + return; } + + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, relationAction.apply(fromId)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, relationAction.apply(toId)); } + private void processRelationByDirection(EntitySearchDirection direction, String relationType, EntityId mainId, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5b6596c9c6..dde24d358a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -740,7 +740,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); @@ -752,7 +751,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 38df529c96..23b9fe08e3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,15 +47,14 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; - private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, false, null); + this(tenantId, entityId, event, null, null, null, null, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -64,7 +63,6 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; - this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 83fb158efd..26a64c7f8a 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,7 +130,6 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); - builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -168,7 +167,6 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); - builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bd441dd679..b060b47429 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1296,7 +1296,6 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 14; } message EdgeEventMsgProto { From 78c4892ad4d0c99ce70315821080deaf5bbd7c1a Mon Sep 17 00:00:00 2001 From: dshvaika Date: Wed, 22 Oct 2025 13:17:41 +0300 Subject: [PATCH 071/122] Added Readiness Status for CF state --- ...CalculatedFieldEntityMessageProcessor.java | 6 +++- .../ctx/state/BaseCalculatedFieldState.java | 18 +++++++++-- .../cf/ctx/state/CalculatedFieldState.java | 31 ++++++++++++++++++- .../ctx/state/SingleValueArgumentEntry.java | 3 ++ .../propagation/PropagationArgumentEntry.java | 3 +- .../PropagationCalculatedFieldState.java | 13 ++------ .../GeofencingCalculatedFieldStateTest.java | 6 ++-- .../state/PropagationArgumentEntryTest.java | 16 ---------- .../PropagationCalculatedFieldStateTest.java | 10 +++--- .../state/ScriptCalculatedFieldStateTest.java | 6 ++-- .../state/SimpleCalculatedFieldStateTest.java | 6 ++-- 11 files changed, 72 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 033292c23e..a7fb74f432 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -399,7 +399,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeChecked = false; try { - if (ctx.isInitialized() && state.isReady()) { + if (ctx.isInitialized() && state.getReadinessStatus().status()) { log.trace("[{}][{}] Performing calculation. Updated args: {}", entityId, ctx.getCfId(), updatedArgs); CalculatedFieldResult calculationResult = state.performCalculation(updatedArgs, ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); @@ -415,6 +415,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } else { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + String errorMsg = ctx.isInitialized() ? state.getReadinessStatus().reason() : "Calculated field state is not initialized!"; + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, errorMsg); + } callback.onSuccess(); } } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index a3966d8a73..8a1b7e64e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -107,9 +108,20 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, } @Override - public boolean isReady() { - return arguments.keySet().containsAll(requiredArguments) && - arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + public ReadinessStatus getReadinessStatus() { + List missing = new ArrayList<>(requiredArguments); + missing.removeAll(arguments.keySet()); + if (!missing.isEmpty()) { + return ReadinessStatus.missingRequiredArguments(missing); + } + List emptyArgs = arguments.entrySet().stream() + .filter(e -> e.getValue() == null || e.getValue().isEmpty()) + .map(Map.Entry::getKey) + .toList(); + if (!emptyArgs.isEmpty()) { + return ReadinessStatus.emptyArguments(emptyArgs); + } + return ReadinessStatus.ready(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 28a14c921a..2bfe09b813 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.Nullable; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; @@ -32,6 +33,7 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; import java.io.Closeable; +import java.util.List; import java.util.Map; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -66,7 +68,7 @@ public interface CalculatedFieldState extends Closeable { ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); @JsonIgnore - boolean isReady(); + ReadinessStatus getReadinessStatus(); boolean isSizeExceedsLimit(); @@ -92,4 +94,31 @@ public interface CalculatedFieldState extends Closeable { } } + record ReadinessStatus(boolean status, @Nullable String reason) { + + private static final String MISSING_REQUIRED_ARGUMENTS = "Missing required arguments: "; + private static final String EMPTY_ARGUMENTS = "Empty arguments: "; + + public static ReadinessStatus ready() { + return new ReadinessStatus(true, null); + } + + public static ReadinessStatus notReady(String reason) { + return new ReadinessStatus(false, reason); + } + + public static ReadinessStatus missingRequiredArguments(List missingArgument) { + return notReady(MISSING_REQUIRED_ARGUMENTS + stringValue(missingArgument)); + } + + private static String stringValue(List missingArgument) { + return String.join(", ", missingArgument); + } + + public static ReadinessStatus emptyArguments(List emptyArguments) { + return notReady(EMPTY_ARGUMENTS + stringValue(emptyArguments)); + } + + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5c1ed32e1d..9fd2ad1662 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -95,6 +95,9 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { + if (isEmpty()) { + return new TbelCfSingleValueArg(ts, null); + } Object value = kvEntryValue.getValue(); if (kvEntryValue instanceof JsonDataEntry) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java index c7d49a4d40..81009da5e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import java.util.ArrayList; import java.util.List; @Data @@ -33,7 +34,7 @@ public class PropagationArgumentEntry implements ArgumentEntry { private boolean forceResetPrevious; public PropagationArgumentEntry(List propagationEntityIds) { - this.propagationEntityIds = propagationEntityIds; + this.propagationEntityIds = new ArrayList<>(propagationEntityIds); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java index 01e9a73de8..cc22797593 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java @@ -33,6 +33,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import java.util.ArrayList; import java.util.Map; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; @@ -47,21 +48,13 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { this.ctx = ctx; this.actorCtx = actorCtx; - this.requiredArguments = ctx.getArgNames(); + this.requiredArguments = new ArrayList<>(ctx.getArgNames()); + requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT); if (ctx.isApplyExpressionForResolvedArguments()) { this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression()); } } - @Override - public boolean isReady() { - if (!super.isReady()) { - return false; - } - ArgumentEntry propagationArg = arguments.get(PROPAGATION_CONFIG_ARGUMENT); - return propagationArg != null && !propagationArg.isEmpty(); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.PROPAGATION; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index 5ca68d4e1b..fd41d633ef 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -199,7 +199,7 @@ public class GeofencingCalculatedFieldStateTest { @Test void testIsReadyWhenNotAllArgPresent() { - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test @@ -210,7 +210,7 @@ public class GeofencingCalculatedFieldStateTest { "allowedZones", geofencingAllowedZoneArgEntry, "restrictedZones", geofencingRestrictedZoneArgEntry )); - assertThat(state.isReady()).isTrue(); + assertThat(state.getReadinessStatus().status()).isTrue(); } @Test @@ -224,7 +224,7 @@ public class GeofencingCalculatedFieldStateTest { state.getArguments().put("noParkingZones", new GeofencingArgumentEntry()); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java index 9c8a788e15..14a1b629c1 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java @@ -57,12 +57,6 @@ public class PropagationArgumentEntryTest { assertThat(emptyEntry.isEmpty()).isTrue(); } - @Test - void testIsEmptyWhenNullList() { - PropagationArgumentEntry nullListEntry = new PropagationArgumentEntry(null); - assertThat(nullListEntry.isEmpty()).isTrue(); - } - @Test void testGetValueReturnsPropagationIds() { assertThat(entry.getValue()).isInstanceOf(List.class); @@ -106,16 +100,6 @@ public class PropagationArgumentEntryTest { assertThat(entry.getPropagationEntityIds()).isEmpty(); } - @Test - void testUpdateEntryClearsWhenNewEntryIsNullList() { - var updatedNull = new PropagationArgumentEntry(null); - - boolean changed = entry.updateEntry(updatedNull); - - assertThat(changed).isTrue(); - assertThat(entry.getPropagationEntityIds()).isEmpty(); - } - @Test @SuppressWarnings("unchecked") void testToTbelCfArgWithValues() { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java index 04a7ab5203..ac9fdefff6 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java @@ -116,20 +116,20 @@ public class PropagationCalculatedFieldStateTest { @Test void testInitAddsRequiredArgument() { initCtxAndState(false); - assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME); + assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT); } @Test void testIsReadyReturnFalseWhenNoArgumentsSet() { initCtxAndState(false); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test void testIsReadyWhenPropagationArgIsNull() { initCtxAndState(false); state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test @@ -137,7 +137,7 @@ public class PropagationCalculatedFieldStateTest { initCtxAndState(false); state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test @@ -145,7 +145,7 @@ public class PropagationCalculatedFieldStateTest { initCtxAndState(false); state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); - assertThat(state.isReady()).isTrue(); + assertThat(state.getReadinessStatus().status()).isTrue(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index e46f3e1c15..8db34a884f 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -160,21 +160,21 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenNotAllArgPresent() { - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test void testIsReadyWhenAllArgPresent() { state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); - assertThat(state.isReady()).isTrue(); + assertThat(state.getReadinessStatus().status()).isTrue(); } @Test void testIsReadyWhenEmptyEntryPresents() { state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } private TsRollingArgumentEntry createRollingArgEntry() { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index df8bf1fbba..6af253ff1b 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -203,7 +203,7 @@ public class SimpleCalculatedFieldStateTest { @Test void testIsReadyWhenNotAllArgPresent() { - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } @Test @@ -214,7 +214,7 @@ public class SimpleCalculatedFieldStateTest { "key3", key3ArgEntry )); - assertThat(state.isReady()).isTrue(); + assertThat(state.getReadinessStatus().status()).isTrue(); } @Test @@ -225,7 +225,7 @@ public class SimpleCalculatedFieldStateTest { )); state.getArguments().put("key3", new SingleValueArgumentEntry()); - assertThat(state.isReady()).isFalse(); + assertThat(state.getReadinessStatus().status()).isFalse(); } private CalculatedField getCalculatedField() { From f48b8752d225e31f85b2acd745c8d5babc21bc0d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 14:33:12 +0300 Subject: [PATCH 072/122] fixed agg cfs filtration in queueu service --- .../service/cf/DefaultCalculatedFieldQueueService.java | 9 ++++++++- .../service/cf/ctx/state/CalculatedFieldState.java | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 84f49c7c9e..c7e369a862 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -199,7 +199,14 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); if (!byRelationPathQuery.isEmpty()) { - return true; + EntityId cfEntityId = cfCtx.getEntityId(); + for (EntityRelation entityRelation : byRelationPathQuery) { + EntityId relatedId = (inverseDirection == EntitySearchDirection.FROM) ? entityRelation.getTo() : entityRelation.getFrom(); + if (cfEntityId.equals(relatedId) || cfEntityId.equals(calculatedFieldCache.getProfileId(tenantId, relatedId))) { + return true; + } + } + return false; } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 2b3ba19528..34e9dad439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -55,8 +55,6 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); - CalculatedFieldCtx getCtx(); - void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); From 269794ec27d9faa2452c3761e11908968f25f95b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 15:54:33 +0300 Subject: [PATCH 073/122] added json subtype --- .../server/service/cf/ctx/state/CalculatedFieldState.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 34e9dad439..9598cc2b49 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -42,7 +43,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldState.class, name = "ALARM"), - @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION") + @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION"), + @Type(value = RelatedEntitiesAggregationCalculatedFieldState.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface CalculatedFieldState extends Closeable { From 4c656fe89d19a35c505a2065343478b722aee163 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 22 Oct 2025 15:59:31 +0300 Subject: [PATCH 074/122] Alarm rules CF: refactoring and improvements --- .../server/actors/ActorSystemContext.java | 5 +- ...CalculatedFieldEntityMessageProcessor.java | 32 +++++----- ...alculatedFieldManagerMessageProcessor.java | 34 ++++++++-- .../cf/ctx/state/CalculatedFieldCtx.java | 56 ++++++++++------- .../alarm/AlarmCalculatedFieldState.java | 14 ++++- .../cf/ctx/state/alarm/AlarmRuleState.java | 22 +++++++ .../entitiy/EntityStateSourcingListener.java | 1 - ...faultTbCalculatedFieldConsumerService.java | 18 +----- .../thingsboard/server/cf/AlarmRulesTest.java | 63 +++++++++++++------ .../src/test/resources/logback-test.xml | 2 - .../server/dao/alarm/AlarmService.java | 4 -- .../common/data/alarm/rule/AlarmRule.java | 2 + .../alarm/rule/condition/AlarmCondition.java | 1 + .../rule/condition/AlarmConditionValue.java | 8 +++ .../expression/AlarmConditionFilter.java | 2 - .../predicate/BooleanFilterPredicate.java | 5 ++ .../predicate/NumericFilterPredicate.java | 5 ++ .../predicate/StringFilterPredicate.java | 6 ++ .../AlarmCalculatedFieldConfiguration.java | 35 +++++++++-- .../CalculatedFieldConfiguration.java | 3 +- .../data/event/CalculatedFieldDebugEvent.java | 2 +- .../common/data/util/CollectionsUtil.java | 29 +++++++++ common/proto/src/main/proto/queue.proto | 2 - .../server/dao/alarm/BaseAlarmService.java | 2 +- .../rule/engine/profile/AlarmRuleState.java | 1 - .../engine/profile/TbDeviceProfileNode.java | 4 +- 26 files changed, 253 insertions(+), 105 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index bf84163a8b..35cf9cb467 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -860,8 +860,9 @@ public class ActorSystemContext { if (errorMessage != null) { eventBuilder.error(errorMessage); } - - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + CalculatedFieldDebugEvent event = eventBuilder.build(); + log.debug("Persisting calculated field debug event: {}", event); + ListenableFuture future = eventService.saveAsync(event); Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } catch (IllegalArgumentException ex) { log.warn("Failed to persist calculated field debug message", ex); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 033292c23e..182d815c96 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -174,7 +174,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); var ctx = msg.getCtx(); - var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); try { Map dynamicSourceArgs = ctx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().hasOwnerSource()) @@ -183,7 +182,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, dynamicSourceArgs); fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); - processArgumentValuesUpdate(ctx, Collections.singletonList(ctx.getCfId()), callback, fetchedArgs, null, null); + processArgumentValuesUpdate(ctx, Collections.singletonList(ctx.getCfId()), msg.getCallback(), fetchedArgs, null, null); } catch (Exception e) { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } @@ -213,7 +212,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); - var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + var numberOfCallbacks = msg.getEntityIdFields().size() + msg.getProfileIdFields().size(); MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); List cfIdList = getCalculatedFieldIds(proto); Set cfIdSet = new HashSet<>(cfIdList); @@ -229,11 +228,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.trace("[{}] Processing CF link telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); var ctx = msg.getCtx(); - var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + var callback = msg.getCallback(); try { List cfIds = getCalculatedFieldIds(proto); if (cfIds.contains(ctx.getCfId())) { - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } else { if (proto.getTsDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); @@ -244,7 +243,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else if (proto.getRemovedAttrKeysCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else { - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } } } catch (Exception e) { @@ -253,10 +252,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, TbCallback callback) throws CalculatedFieldException { try { if (cfIds.contains(ctx.getCfId())) { - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } else { if (proto.getTsDataCount() > 0) { processTelemetry(ctx, proto, cfIdList, callback); @@ -267,7 +266,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else if (proto.getRemovedAttrKeysCount() > 0) { processRemovedAttributes(ctx, proto, cfIdList, callback); } else { - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } } } catch (Exception e) { @@ -307,27 +306,27 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); } - private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, TbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, TbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, TbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } - private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, TbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } - private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, TbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { if (newArgValues.isEmpty()) { log.debug("[{}] No new argument values to process for CF.", ctx.getCfId()); - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } CalculatedFieldState state = states.get(ctx.getCfId()); boolean justRestored = false; @@ -354,7 +353,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM cfIdList.add(ctx.getCfId()); processStateIfReady(state, updatedArgs, ctx, cfIdList, tbMsgId, tbMsgType, callback); } else { - callback.onSuccess(CALLBACKS_PER_CF); + callback.onSuccess(); } } else { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); @@ -395,6 +394,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldState state, Map updatedArgs, CalculatedFieldCtx ctx, List cfIdList, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + callback = new MultipleTbCallback(CALLBACKS_PER_CF, callback); log.trace("[{}][{}] Processing state if ready. Current args: {}, updated args: {}", entityId, ctx.getCfId(), state.getArguments(), updatedArgs); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeChecked = false; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4675821a5b..5d7831eaba 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -121,7 +121,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void stop() { log.info("[{}] Stopping CF manager actor.", tenantId); - calculatedFields.values().forEach(CalculatedFieldCtx::stop); + calculatedFields.values().forEach(CalculatedFieldCtx::close); calculatedFields.clear(); entityIdCalculatedFields.clear(); entityIdCalculatedFieldLinks.clear(); @@ -326,7 +326,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.debug("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = getCfCtx(newCf); // fixme wtf? why isn't oldCfCtx closed properly? when to close it? + var newCfCtx = getCfCtx(newCf); try { newCfCtx.init(); } catch (Exception e) { @@ -366,14 +366,26 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return; } - applyToTargetCfEntityActors(newCfCtx, callback, (id, cb) -> initCfForEntity(id, newCfCtx, stateAction, cb)); + applyToTargetCfEntityActors(newCfCtx, new TbCallback() { + @Override + public void onSuccess() { + oldCfCtx.close(); + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + oldCfCtx.close(); + callback.onFailure(t); + } + }, (id, cb) -> initCfForEntity(id, newCfCtx, stateAction, cb)); } } } private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? + var cfCtx = calculatedFields.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -381,7 +393,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); deleteLinks(cfCtx); - applyToTargetCfEntityActors(cfCtx, callback, (id, cb) -> deleteCfForEntity(id, cfId, cb)); + applyToTargetCfEntityActors(cfCtx, new TbCallback() { + @Override + public void onSuccess() { + cfCtx.close(); + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + cfCtx.close(); + callback.onFailure(t); + } + }, (id, cb) -> deleteCfForEntity(id, cfId, cb)); } public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 838e2d5e2c..98b8e184d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -58,6 +58,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; +import java.io.Closeable; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -66,11 +67,10 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; @Data @Slf4j -public class CalculatedFieldCtx { +public class CalculatedFieldCtx implements Closeable { private CalculatedField calculatedField; @@ -197,15 +197,12 @@ public class CalculatedFieldCtx { } case ALARM -> { AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); - Stream rules = configuration.getCreateRules().values().stream(); - if (configuration.getClearRule() != null) { - rules = Stream.concat(rules, Stream.of(configuration.getClearRule())); - } - rules.map(rule -> rule.getCondition().getExpression()).forEach(expression -> { - if (expression instanceof TbelAlarmConditionExpression tbelExpression) { - initTbelExpression(tbelExpression.getExpression()); - } - }); + configuration.getAllRules().map(rule -> rule.getValue().getCondition().getExpression()) + .forEach(expression -> { + if (expression instanceof TbelAlarmConditionExpression tbelExpression) { + initTbelExpression(tbelExpression.getExpression()); + } + }); initialized = true; } case PROPAGATION -> { @@ -259,7 +256,6 @@ public class CalculatedFieldCtx { public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); - // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg return systemContext.scheduleMsgWithDelay(actorCtx, new CalculatedFieldReevaluateMsg(tenantId, this), delayMs); } @@ -508,8 +504,17 @@ public class CalculatedFieldCtx { if (!Objects.equals(output, other.output)) { return true; } - if (cfType == CalculatedFieldType.ALARM && !calculatedField.getName().equals(other.getCalculatedField().getName())) { - return true; + if (cfType == CalculatedFieldType.ALARM) { + if (!calculatedField.getName().equals(other.getCalculatedField().getName())) { + return true; + } + + var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); + var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); + if (!thisConfig.rulesEqual(otherConfig, AlarmRule::equals)) { + // if the rules have any changes not tracked by hasStateChanges + return true; + } } return scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis; } @@ -521,8 +526,10 @@ public class CalculatedFieldCtx { if (cfType == CalculatedFieldType.ALARM) { var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); - if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || - !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { + if (!thisConfig.rulesEqual(otherConfig, (thisRule, otherRule) -> { + return thisRule.getCondition().getType() == otherRule.getCondition().getType(); + })) { + // reinitializing only if the rule list changed, or if a condition type changed for any rule return true; } } @@ -562,12 +569,17 @@ public class CalculatedFieldCtx { }; } - public void stop() { - if (tbelExpressions != null) { - tbelExpressions.values().forEach(CalculatedFieldScriptEngine::destroy); - } - if (simpleExpressions != null) { - simpleExpressions.values().forEach(ThreadLocal::remove); + @Override + public void close() { + try { + if (tbelExpressions != null) { + tbelExpressions.values().forEach(CalculatedFieldScriptEngine::destroy); + } + if (simpleExpressions != null) { + simpleExpressions.values().forEach(ThreadLocal::remove); + } + } catch (Exception e) { + log.warn("Failed to stop {}", this, e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 02f1725cf2..518121a2d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -103,6 +103,18 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { this.configuration = getConfiguration(ctx); this.alarmType = ctx.getCalculatedField().getName(); + Map createRules = configuration.getCreateRules(); + createRules.forEach((severity, rule) -> { + AlarmRuleState ruleState = createRuleStates.get(severity); + if (ruleState != null) { + ruleState.setAlarmRule(rule); + } + }); + AlarmRule clearRule = configuration.getClearRule(); + if (clearRule != null && clearRuleState != null) { + clearRuleState.setAlarmRule(clearRule); + } + if (currentAlarm != null && !currentAlarm.getType().equals(alarmType)) { currentAlarm = null; initialFetchDone = false; @@ -265,7 +277,7 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { clearState(state); } AlarmApiCallResult clearResult = ctx.getAlarmService().clearAlarm( - ctx.getTenantId(), currentAlarm.getId(), System.currentTimeMillis(), createDetails(clearRuleState), true + ctx.getTenantId(), currentAlarm.getId(), System.currentTimeMillis(), createDetails(clearRuleState), false ); if (clearResult.isCleared()) { result = TbAlarmResult.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java index 9c1b966878..8612607dfb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmRuleState.java @@ -237,7 +237,15 @@ public class AlarmRuleState { } public void clear() { + clearRepeatingConditionState(); + clearDurationConditionState(); + } + + private void clearRepeatingConditionState() { eventCount = 0L; + } + + private void clearDurationConditionState() { firstEventTs = 0L; lastEventTs = 0L; duration = 0L; @@ -289,6 +297,20 @@ public class AlarmRuleState { public void setAlarmRule(AlarmRule alarmRule) { this.alarmRule = alarmRule; this.condition = alarmRule.getCondition(); + + // clearing state for other condition types (possibly left from a previous condition type) + switch (condition.getType()) { + case SIMPLE -> { + clearRepeatingConditionState(); + clearDurationConditionState(); + } + case REPEATING -> { + clearDurationConditionState(); + } + case DURATION -> { + clearRepeatingConditionState(); + } + } } public StateInfo getStateInfo() { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index b9fd38f1e2..2589f67401 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -255,7 +255,6 @@ public class EntityStateSourcingListener { if (calculatedFieldCache.hasCalculatedFields(tenantId, alarm.getOriginator(), ctx -> ctx.getCfType() == CalculatedFieldType.ALARM)) { ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder() .setEventMsg(toProto(event)) - .addCfTypes(CalculatedFieldType.ALARM.name()) .build(); tbClusterService.pushMsgToCalculatedFields(tenantId, alarm.getOriginator(), msg, new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 952ca845f0..0449d116c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PreDestroy; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; @@ -28,7 +27,6 @@ import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTeleme import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -61,7 +59,6 @@ import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedCon import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; -import java.util.EnumSet; import java.util.List; import java.util.Set; import java.util.UUID; @@ -183,8 +180,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa } private void processMsg(ToCalculatedFieldMsg toCfMsg, UUID id, TbCallback callback) { - Set cfTypes = getCfTypes(toCfMsg.getCfTypesList()); - if (toCfMsg.hasTelemetryMsg()) { // TODO: add CF type filter to the message. or just rename the CF strategy to "Process alarms and calculated fields + if (toCfMsg.hasTelemetryMsg()) { log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); } else if (toCfMsg.hasLinkedTelemetryMsg()) { @@ -264,18 +260,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); } - private Set getCfTypes(ProtocolStringList cfTypesList) { - Set cfTypes; - if (cfTypesList.isEmpty()) { - cfTypes = EnumSet.allOf(CalculatedFieldType.class); - } else { - cfTypes = cfTypesList.stream() - .map(CalculatedFieldType::valueOf) - .collect(Collectors.toSet()); - } - return cfTypes; - } - @Override protected void stopConsumers() { super.stopConsumers(); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 2b43373ee5..f71bdd02a8 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -458,7 +458,14 @@ public class AlarmRulesTest extends AbstractControllerTest { schedule = schedule.replace("\"enabled\":false", "\"enabled\":true"); postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"schedule\":" + schedule + "}"); - checkAlarmResult(calculatedField, alarmResult -> { + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + // checking multiple debug events due to scheduled reevaluation (which also produces debug events) + CalculatedFieldDebugEvent debugEvent = getDebugEvents(calculatedField.getId(), 5).stream() + .filter(event -> event.getResult() != null) + .findFirst().orElse(null); + assertThat(debugEvent).isNotNull(); + TbAlarmResult alarmResult = JacksonUtil.fromString(debugEvent.getResult(), TbAlarmResult.class); assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); @@ -638,11 +645,11 @@ public class AlarmRulesTest extends AbstractControllerTest { AlarmSeverity.CRITICAL, new Condition("return temperature >= 50 && humidity >= 50;", null, null) ); CalculatedField calculatedField = createAlarmCf(deviceId, "High Temperature and Humidity Alarm", - arguments, createRules, null); - AlarmCalculatedFieldConfiguration configuration = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); - configuration.getCreateRules().get(AlarmSeverity.CRITICAL).setAlarmDetails(""" - temperature is ${temperature}, humidity is ${humidity}"""); - calculatedField = saveCalculatedField(calculatedField); + arguments, createRules, null, configuration -> { + configuration.getCreateRules().get(AlarmSeverity.CRITICAL).setAlarmDetails( + "temperature is ${temperature}, humidity is ${humidity}" + ); + }); postTelemetry(deviceId, "{\"temperature\":50}"); postAttributes(deviceId, AttributeScope.SERVER_SCOPE, "{\"humidity\":50}"); @@ -653,6 +660,18 @@ public class AlarmRulesTest extends AbstractControllerTest { assertThat(alarmResult.getAlarm().getDetails().get("data").asText()) .isEqualTo("temperature is 50, humidity is 50"); }); + + ((AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration()).getCreateRules().get(AlarmSeverity.CRITICAL).setAlarmDetails( + "UPDATED temperature is ${temperature}, humidity is ${humidity}" + ); + calculatedField = saveCalculatedField(calculatedField); + checkAlarmResult(calculatedField, alarmResult -> { + assertThat(alarmResult.isCreated()).isFalse(); + assertThat(alarmResult.isUpdated()).isTrue(); + assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); + assertThat(alarmResult.getAlarm().getDetails().get("data").asText()) + .isEqualTo("UPDATED temperature is 50, humidity is 50"); + }); } @Test @@ -683,7 +702,12 @@ public class AlarmRulesTest extends AbstractControllerTest { Thread.sleep(10000); assertThat(getLatestAlarmResult(calculatedField.getId())).isNull(); - checkAlarmResult(calculatedField, alarmResult -> { + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + CalculatedFieldDebugEvent debugEvent = getDebugEvents(calculatedField.getId(), 5).stream() + .filter(event -> event.getResult() != null) + .findFirst().orElse(null); + assertThat(debugEvent).isNotNull(); + TbAlarmResult alarmResult = JacksonUtil.fromString(debugEvent.getResult(), TbAlarmResult.class); assertThat(alarmResult.isCreated()).isTrue(); assertThat(alarmResult.getAlarm().getSeverity()).isEqualTo(AlarmSeverity.CRITICAL); assertThat(alarmResult.getAlarm().getStatus()).isEqualTo(AlarmStatus.ACTIVE_UNACK); @@ -764,24 +788,14 @@ public class AlarmRulesTest extends AbstractControllerTest { String alarmType, Map arguments, Map createConditions, - Condition clearCondition) { + Condition clearCondition, + Consumer... modifier) { Map createRules = new HashMap<>(); createConditions.forEach((severity, condition) -> { createRules.put(severity, toAlarmRule(condition)); }); AlarmRule clearRule = clearCondition != null ? toAlarmRule(clearCondition) : null; - CalculatedField calculatedField = createAlarmCf(entityId, alarmType, arguments, createRules, clearRule); - - CalculatedFieldDebugEvent debugEvent = await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getDebugEvents(calculatedField.getId(), 1), events -> !events.isEmpty()).get(0); - latestEventId = debugEvent.getId(); - return calculatedField; - } - private CalculatedField createAlarmCf(EntityId entityId, - String alarmType, - Map arguments, - Map createRules, - AlarmRule clearRule) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(entityId); calculatedField.setName(alarmType); @@ -792,7 +806,16 @@ public class AlarmRulesTest extends AbstractControllerTest { configuration.setClearRule(clearRule); calculatedField.setConfiguration(configuration); calculatedField.setDebugSettings(DebugSettings.all()); - return saveCalculatedField(calculatedField); + if (modifier.length > 0) { + modifier[0].accept(configuration); + } + CalculatedField savedCalculatedField = saveCalculatedField(calculatedField); + + CalculatedFieldDebugEvent debugEvent = await().atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getDebugEvents(savedCalculatedField.getId(), 1), + events -> !events.isEmpty()).get(0); + latestEventId = debugEvent.getId(); + return savedCalculatedField; } private AlarmRule toAlarmRule(Condition condition) { diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 56dbbfc125..13c93da411 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -17,8 +17,6 @@ - - diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 82ef7f4e8d..05abc4b0c7 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -51,10 +51,6 @@ import java.util.UUID; public interface AlarmService extends EntityDaoService { - /* - * New API, since 3.5. - */ - /** * Designed for atomic operations over active alarms. * Only one active alarm may exist for the pair {originatorId, alarmType} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java index ab7adcbd48..9a4e875154 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/AlarmRule.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.alarm.rule; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import lombok.Data; @@ -30,6 +31,7 @@ public class AlarmRule { private String alarmDetails; private DashboardId dashboardId; + @JsonIgnore public boolean requiresScheduledReevaluation() { return condition.hasSchedule(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java index c9280151d1..9bb549994b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmCondition.java @@ -45,6 +45,7 @@ public abstract class AlarmCondition { @Valid private AlarmConditionValue schedule; + @JsonIgnore public boolean hasSchedule() { return schedule != null && !(schedule.getStaticValue() instanceof AnyTimeSchedule); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java index 84a1498ef6..fab3a78ab3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/AlarmConditionValue.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition; +import com.fasterxml.jackson.annotation.JsonIgnore; +import jakarta.validation.constraints.AssertTrue; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -27,4 +29,10 @@ public class AlarmConditionValue { private T staticValue; private String dynamicValueArgument; + @JsonIgnore + @AssertTrue(message = "Either staticValue or dynamicValueArgument must be set") + public boolean isValid() { + return staticValue != null ^ dynamicValueArgument != null; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java index aa70feca13..e9785d675b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition.expression; -import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -24,7 +23,6 @@ import org.thingsboard.server.common.data.alarm.rule.condition.expression.predic import java.io.Serializable; -@Schema @Data public class AlarmConditionFilter implements Serializable { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java index 8a57aba3e0..94dced5fe4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/BooleanFilterPredicate.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; @Data public class BooleanFilterPredicate implements SimpleKeyFilterPredicate { + @NotNull private BooleanOperation operation; + @Valid + @NotNull private AlarmConditionValue value; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java index 30a82e06bb..65316eda88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/NumericFilterPredicate.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; @Data public class NumericFilterPredicate implements SimpleKeyFilterPredicate { + @NotNull private NumericOperation operation; + @Valid + @NotNull private AlarmConditionValue value; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java index ccc263611f..913c12ca1c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/predicate/StringFilterPredicate.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.alarm.rule.condition.AlarmConditionValue; @Data public class StringFilterPredicate implements SimpleKeyFilterPredicate { + @NotNull private StringOperation operation; + @Valid + @NotNull private AlarmConditionValue value; private boolean ignoreCase; @@ -40,4 +45,5 @@ public class StringFilterPredicate implements SimpleKeyFilterPredicate { IN, NOT_IN } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java index 900973ac1a..d36ba33849 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/AlarmCalculatedFieldConfiguration.java @@ -15,15 +15,24 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import lombok.Data; +import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.rule.AlarmRule; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.BiPredicate; +import java.util.stream.Stream; + +import static java.util.Map.Entry.comparingByKey; @Data public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { @@ -51,15 +60,31 @@ public class AlarmCalculatedFieldConfiguration implements ArgumentsBasedCalculat return null; } + @JsonIgnore @Override - public void validate() { + public boolean requiresScheduledReevaluation() { + return getAllRules().anyMatch(entry -> entry.getValue().requiresScheduledReevaluation()); + } + @JsonIgnore + public Stream> getAllRules() { + Stream> rules = createRules.entrySet().stream() + .map(entry -> Pair.of(entry.getKey(), entry.getValue())); + if (clearRule != null) { + rules = Stream.concat(rules, Stream.of(Pair.of(null, clearRule))); + } + return rules.sorted(comparingByKey(Comparator.nullsLast(Comparator.naturalOrder()))); } - @Override - public boolean requiresScheduledReevaluation() { - return createRules.values().stream().anyMatch(AlarmRule::requiresScheduledReevaluation) || - (clearRule != null && clearRule.requiresScheduledReevaluation()); + public boolean rulesEqual(AlarmCalculatedFieldConfiguration other, BiPredicate equalityCheck) { + List> thisRules = this.getAllRules().toList(); + List> otherRules = other.getAllRules().toList(); + return CollectionsUtil.elementsEqual(thisRules, otherRules, (thisRule, otherRule) -> { + if (!Objects.equals(thisRule.getKey(), otherRule.getKey())) { + return false; + } + return equalityCheck.test(thisRule.getValue(), otherRule.getValue()); + }); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index bdf2bdcb93..7be23f8391 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -50,7 +50,7 @@ public interface CalculatedFieldConfiguration { Output getOutput(); - void validate(); + default void validate() {} @JsonIgnore default List getReferencedEntities() { @@ -72,6 +72,7 @@ public interface CalculatedFieldConfiguration { .collect(Collectors.toList()); } + @JsonIgnore default boolean requiresScheduledReevaluation() { return false; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java index 0424eabeb6..acc5cf6205 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; -@ToString +@ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) public class CalculatedFieldDebugEvent extends Event { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 71c5256203..e92c62242c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -18,9 +18,11 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiPredicate; import java.util.stream.Collectors; public class CollectionsUtil { @@ -95,4 +97,31 @@ public class CollectionsUtil { return false; } + public static boolean elementsEqual(Iterable iterable1, Iterable iterable2, BiPredicate equalityCheck) { + if (iterable1 instanceof Collection collection1 && iterable2 instanceof Collection collection2) { + if (collection1.size() != collection2.size()) { + return false; + } + } + + Iterator iterator1 = iterable1.iterator(); + Iterator iterator2 = iterable2.iterator(); + while (true) { + if (iterator1.hasNext()) { + if (!iterator2.hasNext()) { + return false; + } + + T o1 = iterator1.next(); + T o2 = iterator2.next(); + if (equalityCheck.test(o1, o2)) { + continue; + } else { + return false; + } + } + return !iterator2.hasNext(); + } + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4d99608a6d..8602994c62 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1724,12 +1724,10 @@ message ToCalculatedFieldMsg { CalculatedFieldTelemetryMsgProto telemetryMsg = 1; CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; EntityActionEventProto eventMsg = 3; - repeated string cfTypes = 4; } message ToCalculatedFieldNotificationMsg { CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 1; - repeated string cfTypes = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 3df4a64e73..413bf5ffb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -159,7 +159,7 @@ public class BaseAlarmService extends AbstractCachedEntityService Date: Thu, 23 Oct 2025 11:19:53 +0300 Subject: [PATCH 075/122] scheduling for agg cfs on restart --- .../CalculatedFieldManagerMessageProcessor.java | 5 ++++- .../RelatedEntitiesAggregationCalculatedFieldState.java | 6 ------ ...tedEntitiesAggregationCalculatedFieldConfiguration.java | 7 +++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 40707e32f3..de3967d5b6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -652,7 +652,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { List cfsByEntityIdAndProfile = new ArrayList<>(); cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); - cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(profileId)); + } return cfsByEntityIdAndProfile; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 7e530b6809..655217263b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -75,12 +75,6 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat metrics = null; } - @Override - public void init() { - super.init(); - ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 9d4c7bdaf6..931cb919ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -56,4 +57,10 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A } } + @JsonIgnore + @Override + public boolean requiresScheduledReevaluation() { + return true; + } + } From bb308919ce3d8ad297b79c4288bf93c8f3230289 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 15:43:43 +0300 Subject: [PATCH 076/122] UI: Add new tenant profile configuration minAllowedDeduplicationIntervalInSecForCF --- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 1 + ...ult-tenant-profile-configuration.component.html | 14 +++++++++++++- ...fault-tenant-profile-configuration.component.ts | 1 + ui-ngx/src/app/shared/models/tenant.model.ts | 2 ++ .../src/assets/locale/locale.constant-en_US.json | 3 +++ 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index d6612f2427..6e4d324b5b 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -31,6 +31,7 @@ export interface SysParamsState { maxDebugModeDurationMinutes: number; maxDataPointsPerRollingArg: number; maxArgumentsPerCF: number; + minAllowedDeduplicationIntervalInSecForCF: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxRelationLevelPerCfArgument: number; ruleChainDebugPerTenantLimitsConfiguration?: string; diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 8cfbc04197..777cf5308e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -33,6 +33,7 @@ const emptyUserAuthState: AuthPayload = { mobileQrEnabled: false, maxResourceSize: 0, maxArgumentsPerCF: 0, + minAllowedDeduplicationIntervalInSecForCF: 0, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxRelationLevelPerCfArgument: 0, maxDataPointsPerRollingArg: 0, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index d53cacbd83..8cfa8fd0ed 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -354,7 +354,19 @@ tenant-profile.relation-search-entity-limit-hint -
+ + tenant-profile.min-allowed-deduplication-interval + + + {{ 'tenant-profile.min-allowed-deduplication-interval-required' | translate}} + + + {{ 'tenant-profile.min-allowed-deduplication-interval-range' | translate}} + + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 9595def95e..0000d01995 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -116,6 +116,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxCalculatedFieldsPerEntity: [0, [Validators.required, Validators.min(0)]], maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], + minAllowedDeduplicationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 1ed1092207..ae7a0ae8b8 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -107,6 +107,7 @@ export interface DefaultTenantProfileConfiguration { maxCalculatedFieldsPerEntity: number; maxArgumentsPerCF: number; maxRelationLevelPerCfArgument: number; + minAllowedDeduplicationIntervalInSecForCF: number; maxRelatedEntitiesToReturnPerCfArgument: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; @@ -174,6 +175,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxArgumentsPerCF: 10, maxDataPointsPerRollingArg: 1000, maxRelationLevelPerCfArgument: 10, + minAllowedDeduplicationIntervalInSecForCF: 3600, maxRelatedEntitiesToReturnPerCfArgument: 100, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3ee738a8b0..331f3cdb09 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5876,6 +5876,9 @@ "max-related-level-per-argument-required": "Relation level per 'Related entities' argument max number is required", "min-allowed-scheduled-update-interval": "Min allowed update interval for 'Related entities' arguments (seconds)", "min-allowed-scheduled-update-interval-range": "Min allowed update interval min number can't be negative", + "min-allowed-deduplication-interval": "Min allowed deduplication interval (seconds)", + "min-allowed-deduplication-interval-range": "Min allowed deduplication interval value can't be negative", + "min-allowed-deduplication-interval-required": "Min allowed deduplication interval is required", "min-allowed-scheduled-update-interval-required": "Min allowed update interval min number is required", "max-state-size": "State maximum size in KB", "max-state-size-range": "State maximum size in KB can't be negative", From 59d2fe8c7b40866cd97f58cff2162d9a2aa0e905 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 19:06:55 +0300 Subject: [PATCH 077/122] UI: Add bacis config RELATED_ENTITIES_AGGREGATION cf --- .../calculated-field.module.ts | 4 + ...ulated-field-argument-panel.component.html | 54 +++--- ...lculated-field-argument-panel.component.ts | 11 ++ ...calculated-field-arguments-table.module.ts | 9 +- ...d-aggregation-arguments-table.component.ts | 70 ++++++++ .../calculated-field-dialog.component.html | 7 + .../calculated-field-dialog.component.scss | 3 + .../calculated-field-output.component.html | 85 +++++----- .../calculated-field-output.component.ts | 13 +- ...ities-aggregation-component.component.html | 75 +++++++++ ...ntities-aggregation-component.component.ts | 154 ++++++++++++++++++ ...d-entities-aggregation-component.module.ts | 45 +++++ .../components/time-unit-input.component.html | 6 +- .../shared/models/calculated-field.models.ts | 24 ++- .../assets/locale/locale.constant-en_US.json | 15 +- 15 files changed, 505 insertions(+), 70 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index e0db7a6eef..5e3a854aa2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -38,6 +38,9 @@ import { import { PropagationConfigurationModule } from '@home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module'; +import { + RelatedEntitiesAggregationComponentModule +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module'; @NgModule({ declarations: [ @@ -52,6 +55,7 @@ import { EntityDebugSettingsButtonComponent, SimpleConfigurationModule, PropagationConfigurationModule, + RelatedEntitiesAggregationComponentModule, ], exports: [ CalculatedFieldDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 77bdedb068..607b107094 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -18,6 +18,11 @@
{{ 'calculated-fields.argument-settings' | translate }}
+ @if (hint) { +
+ {{ hint | translate }} +
+ }
@if (!isOutputKey) { } -
-
{{ 'entity.entity-type' | translate }}
- - - @for (type of argumentEntityTypes; track type) { - {{ ArgumentEntityTypeTranslations.get(type) | translate }} + @if (!hiddenEntityTypes) { +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + @if (argumentType.touched && argumentType.hasError('required')) { + + warning + } - - @if (argumentType.touched && argumentType.hasError('required')) { - - warning - - } - -
+ +
+ } @if (ArgumentEntityTypeParamsMap.has(entityType)) {
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
@@ -143,9 +150,18 @@ } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) {
-
{{ 'calculated-fields.default-value' | translate }}
+
{{ 'calculated-fields.default-value' | translate }}
+ @if (argumentFormGroup.get('defaultValue').touched && argumentFormGroup.get('defaultValue').hasError('required')) { + + warning + + }
} @else { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index 070ffb5c06..6f90d126e0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -52,6 +52,7 @@ import { Store } from '@ngrx/store'; import { EntityAutocompleteComponent } from '@shared/components/entity/entity-autocomplete.component'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { TenantId } from '@shared/models/id/tenant-id'; +import { deduplicationStrategiesHintTranslations } from '@home/components/rule-node/rule-node-config.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,6 +69,9 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() isScript: boolean; @Input() usedArgumentNames: string[]; @Input() isOutputKey = false; + @Input() hiddenEntityTypes = false; + @Input() defaultValueRequired = false; + @Input() hint: string; @Input() argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; @ViewChild('entityAutocomplete') entityAutocomplete: EntityAutocompleteComponent; @@ -146,6 +150,11 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.setInitialEntityType(); this.setWatchKeyChange(); + if (this.defaultValueRequired) { + this.argumentFormGroup.get('defaultValue').addValidators(Validators.required); + this.argumentFormGroup.get('defaultValue').updateValueAndValidity({onlySelf: true}); + } + this.argumentTypes = Object.values(ArgumentType) .filter(type => type !== ArgumentType.Rolling || this.isScript); } @@ -311,4 +320,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.entityNameSubject.next(null); } } + + protected readonly deduplicationStrategiesHintTranslations = deduplicationStrategiesHintTranslations; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts index 082001f052..cae0b92387 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts @@ -26,6 +26,9 @@ import { import { PropagateArgumentsTableComponent } from '@home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component'; +import { + RelatedAggregationArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component'; @NgModule({ imports: [ @@ -35,11 +38,13 @@ import { declarations: [ CalculatedFieldArgumentPanelComponent, CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ], exports: [ CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ] }) export class CalculatedFieldArgumentsTableModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts new file mode 100644 index 0000000000..7c9212d3c2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts @@ -0,0 +1,70 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, DestroyRef, forwardRef, Renderer2, ViewContainerRef, } from '@angular/core'; +import { FormBuilder, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from '@angular/forms'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityService } from '@core/http/entity.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { ArgumentEntityType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-related-aggregation-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + } + ], +}) +export class RelatedAggregationArgumentsTableComponent extends CalculatedFieldArgumentsTableComponent { + + constructor( + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store + ) { + super(fb, popoverService, viewContainerRef, cd, renderer, entityService, destroyRef, store); + + this.argumentNameColumn = 'calculated-fields.argument-name'; + this.displayColumns = ['name', 'type', 'key', 'actions']; + this.panelAdditionalCtx = { + hiddenEntityTypes: true, + defaultValueRequired: true, + argumentEntityTypes: [ArgumentEntityType.Current], + hint: 'calculated-fields.hint.setting-arguments-aggregation' + }; + + this.isScript = false; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d9dcc98f1..478d41f42b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -75,6 +75,13 @@ [testScript]="onTestScript.bind(this)"> } + @case (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) { + + + } @default { @if (simpleMode) { -
- - - {{ - (outputForm.get('type').value === OutputType.Timeseries - ? 'calculated-fields.timeseries-key' - : 'calculated-fields.attribute-key') - | translate - }} - - - @if (outputForm.get('name').errors && outputForm.get('name').touched) { - - @if (outputForm.get('name').hasError('required')) { - {{ 'common.hint.key-required' | translate }} - } @else if (outputForm.get('name').hasError('pattern')) { - {{ 'common.hint.key-pattern' | translate }} - } @else if (outputForm.get('name').hasError('maxlength')) { - {{ 'common.hint.key-max-length' | translate }} - } - - } - - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - -
- - - - - - - - - + @if (hiddenName) { +
+ + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + +
+ } @else { +
+ + + {{ + (outputForm.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate + }} + + + @if (outputForm.get('name').errors && outputForm.get('name').touched) { + + @if (outputForm.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputForm.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputForm.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
+ + } }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts index 9a95c921fa..464ca99bd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts @@ -35,6 +35,7 @@ import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { coerceBoolean } from '@shared/decorators/coercion'; @Component({ selector: 'tb-calculate-field-output', @@ -55,8 +56,13 @@ import { EntityType } from '@shared/models/entity-type.models'; export class CalculatedFieldOutputComponent implements ControlValueAccessor, Validator, OnInit, OnChanges { @Input() + @coerceBoolean() simpleMode = false; + @Input() + @coerceBoolean() + hiddenName = false; + @Input({required: true}) entityId: EntityId; @@ -137,11 +143,14 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val } private updatedFormWithMode(): void { - if (this.simpleMode) { + if (this.simpleMode && !this.hiddenName) { this.outputForm.get('name').enable({emitEvent: false}); - this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); } else { this.outputForm.get('name').disable({emitEvent: false}); + } + if (this.simpleMode) { + this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); + } else { this.outputForm.get('decimalsByDefault').disable({emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html new file mode 100644 index 0000000000..1988141c2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -0,0 +1,75 @@ + +
+
+
+ {{ 'calculated-fields.aggregation-path-related-entities' | translate }} +
+
+ + {{ 'calculated-fields.direction' | translate }} + + @for (direction of Directions; track direction) { + {{ PropagationDirectionTranslations.get(direction) | translate }} + } + + + + +
+
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+ +
+
+
+ {{ 'calculated-fields.metrics' | translate }} +
+ + +
+ +
+ +
+ calculated-fields.use-latest-timestamp +
+ +
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts new file mode 100644 index 0000000000..d10372beea --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable, of } from 'rxjs'; +import { + CalculatedFieldOutput, + CalculatedFieldRelatedAggregationConfiguration, + CalculatedFieldType, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + PropagationDirectionTranslations +} from '@shared/models/calculated-field.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@app/shared/models/rule-node.models'; +import { EntitySearchDirection } from '@shared/models/relation.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; + +@Component({ + selector: 'tb-related-entities-aggregation-component', + templateUrl: './related-entities-aggregation-component.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + } + ], +}) +export class RelatedEntitiesAggregationComponentComponent implements ControlValueAccessor, Validator { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + relatedAggregationConfiguration = this.fb.group({ + relation: this.fb.group({ + direction: [EntitySearchDirection.FROM, Validators.required], + relationType: ['Contains', Validators.required], + }), + arguments: this.fb.control({}), + deduplicationIntervalInSec: [], + output: this.fb.control({ + scope: AttributeScope.SERVER_SCOPE, + type: OutputType.Timeseries, + }), + useLatestTs: [false] + }); + + readonly ScriptLanguage = ScriptLanguage; + readonly CalculatedFieldType = CalculatedFieldType; + readonly OutputType = OutputType; + readonly Directions = Object.values(EntitySearchDirection) as Array; + readonly PropagationDirectionTranslations = PropagationDirectionTranslations; + readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; + + + functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj ?? {})) + ); + + argumentsHighlightRules$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + private propagateChange: (config: CalculatedFieldRelatedAggregationConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder, + private store: Store) { + + this.relatedAggregationConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value: CalculatedFieldRelatedAggregationConfiguration) => { + this.updatedModel(value); + }) + } + + validate(): ValidationErrors | null { + return this.relatedAggregationConfiguration.valid || this.relatedAggregationConfiguration.status === "DISABLED" ? null : {invalidPropagateConfig: false}; + } + + writeValue(value: CalculatedFieldRelatedAggregationConfiguration): void { + this.relatedAggregationConfiguration.patchValue(value, {emitEvent: false}); + setTimeout(() => { + this.relatedAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); + } + + registerOnChange(fn: (config: CalculatedFieldRelatedAggregationConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.relatedAggregationConfiguration.disable({emitEvent: false}); + } else { + this.relatedAggregationConfiguration.enable({emitEvent: false}); + } + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(['Contains', 'Manages']).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private updatedModel(value: CalculatedFieldRelatedAggregationConfiguration): void { + value.type = CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + this.propagateChange(value); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts new file mode 100644 index 0000000000..a2c48c1896 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -0,0 +1,45 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; +import { + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; +import { + RelatedEntitiesAggregationComponentComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, + ], + declarations: [ + RelatedEntitiesAggregationComponentComponent, + ], + exports: [ + RelatedEntitiesAggregationComponentComponent, + ] +}) +export class RelatedEntitiesAggregationComponentModule { +} diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.html b/ui-ngx/src/app/shared/components/time-unit-input.component.html index d5f6319576..53a444895b 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.html +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.html @@ -18,7 +18,7 @@
+ subscriptSizing="dynamic"> @if (labelText && !inlineField) { {{ labelText }} } @@ -41,7 +41,9 @@ {{ hasError }} - @if (!inlineField) { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8294a776df..8ea6659bca 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -65,7 +65,8 @@ export enum CalculatedFieldType { SIMPLE = 'SIMPLE', SCRIPT = 'SCRIPT', GEOFENCING = 'GEOFENCING', - PROPAGATION = 'PROPAGATION' + PROPAGATION = 'PROPAGATION', + RELATED_ENTITIES_AGGREGATION = 'RELATED_ENTITIES_AGGREGATION' } export const CalculatedFieldTypeTranslations = new Map( @@ -74,6 +75,7 @@ export const CalculatedFieldTypeTranslations = new Map; + useLatestTs: boolean; output: CalculatedFieldSimpleOutput; } @@ -105,6 +109,15 @@ export interface CalculatedFieldGeofencingConfiguration { output: CalculatedFieldOutput; } +export interface CalculatedFieldRelatedAggregationConfiguration { + type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + relation: RelationPathLevel; + arguments: Record; + deduplicationIntervalInSec: number; + useLatestTs: boolean; + output: Omit; +} + interface BasePropagationConfiguration { type: CalculatedFieldType.PROPAGATION; direction: EntitySearchDirection; @@ -250,7 +263,7 @@ export interface CalculatedFieldGeofencing { export interface RefDynamicSourceConfiguration { type?: ArgumentEntityType.RelationQuery; - levels?: Array<{direction: EntitySearchDirection; relationType: string;}>; + levels?: Array; } export interface CalculatedFieldGeofencingValue extends CalculatedFieldGeofencing { @@ -317,6 +330,11 @@ export interface CalculatedFieldArgumentValueBase { type: ArgumentType; } +export interface RelationPathLevel { + direction: EntitySearchDirection; + relationType: string; +} + export interface CalculatedFieldAttributeArgumentValue extends CalculatedFieldArgumentValueBase { ts: number; value: ValueType; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 331f3cdb09..0333c5ed2e 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1054,7 +1054,8 @@ "simple": "Simple", "script": "Script", "geofencing" : "Geofencing", - "propagation": "Propagation" + "propagation": "Propagation", + "related-entities-aggregation": "Related entities aggregation" }, "arguments": "Arguments", "decimals-by-default": "Decimals by default", @@ -1088,6 +1089,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", + "default-value-required": "Default value is required.", "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", @@ -1156,6 +1158,11 @@ "data-propagate": "Data to propagate", "output-key": "Output key", "copy-output-key": "Copy output key", + "aggregation-path-related-entities": "Aggregation path to related entities", + "metrics": "Metrics", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", + "deduplication-interval-required": "Deduplication interval is required.", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1202,7 +1209,11 @@ "zone-group-refresh-interval-required": "Zone groups refresh interval is required.", "zone-group-refresh-interval-min": "Zone group refresh interval should be at least {{ min }} second.", "propagation-path-related-entities": "Defines a direct, single-level path to a related entity based on the selected direction and relation type.", - "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data." + "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data.", + "aggregation-path-related-entities": "Defines a single-level aggregation path via direct relations with parent or child entities based on direction and relation type. Only relations between device, asset, customer, and tenant entities are supported.", + "arguments-aggregation": "Defines input parameters used for filtering and aggregation.", + "setting-arguments-aggregation": "Data will be fetched from related entities configured in aggregation path.", + "metrics": "Defines metrics aggregated based on the configured arguments." } }, "ai-models": { From aee4b1cee7f1ca49173bd707b62ec245adc7c382 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 24 Oct 2025 13:39:50 +0300 Subject: [PATCH 078/122] Alarm rules CF: add value type field for condition filter --- .../service/install/DefaultSystemDataLoaderService.java | 7 +++++++ .../java/org/thingsboard/server/cf/AlarmRulesTest.java | 3 +++ .../rule/condition/expression/AlarmConditionFilter.java | 3 +++ 3 files changed, 13 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 287581d297..9c285fa9ee 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -82,6 +82,7 @@ import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.queue.ProcessingStrategy; import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; @@ -456,6 +457,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter temperatureAlarmFlagFilter = new AlarmConditionFilter(); temperatureAlarmFlagFilter.setArgument("temperatureAlarmFlag"); + temperatureAlarmFlagFilter.setValueType(EntityKeyValueType.BOOLEAN); BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate(); temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); temperatureAlarmFlagAttributePredicate.setValue(new AlarmConditionValue<>(Boolean.TRUE, null)); @@ -463,6 +465,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); temperatureFilter.setArgument("temperature"); + temperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate temperatureFilterPredicate = new NumericFilterPredicate(); temperatureFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); temperatureFilterPredicate.setValue(new AlarmConditionValue<>(null, "temperatureAlarmThreshold")); @@ -479,6 +482,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter clearTemperatureFilter = new AlarmConditionFilter(); clearTemperatureFilter.setArgument("temperature"); + clearTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate clearTemperatureFilterPredicate = new NumericFilterPredicate(); clearTemperatureFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL); clearTemperatureFilterPredicate.setValue(new AlarmConditionValue<>(null, "temperatureAlarmThreshold")); @@ -517,6 +521,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter humidityAlarmFlagAttributeFilter = new AlarmConditionFilter(); humidityAlarmFlagAttributeFilter.setArgument("humidityAlarmFlag"); + humidityAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); BooleanFilterPredicate humidityAlarmFlagPredicate = new BooleanFilterPredicate(); humidityAlarmFlagPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); humidityAlarmFlagPredicate.setValue(new AlarmConditionValue<>(Boolean.TRUE, null)); @@ -524,6 +529,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter humidityFilter = new AlarmConditionFilter(); humidityFilter.setArgument("humidity"); + humidityFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate humidityFilterPredicate = new NumericFilterPredicate(); humidityFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); humidityFilterPredicate.setValue(new AlarmConditionValue<>(null, "humidityAlarmThreshold")); @@ -540,6 +546,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmConditionFilter clearHumidityFilter = new AlarmConditionFilter(); clearHumidityFilter.setArgument("humidity"); + clearHumidityFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate clearHumidityFilterPredicate = new NumericFilterPredicate(); clearHumidityFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL); clearHumidityFilterPredicate.setValue(new AlarmConditionValue<>(null, "humidityAlarmThreshold")); diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index f71bdd02a8..652e69781d 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -62,6 +62,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -167,6 +168,7 @@ public class AlarmRulesTest extends AbstractControllerTest { SimpleAlarmConditionExpression simpleExpression = new SimpleAlarmConditionExpression(); AlarmConditionFilter filter = new AlarmConditionFilter(); filter.setArgument("temperature"); + filter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setOperation(NumericOperation.GREATER_OR_EQUAL); AlarmConditionValue thresholdValue = new AlarmConditionValue<>(); @@ -854,6 +856,7 @@ public class AlarmRulesTest extends AbstractControllerTest { SimpleAlarmConditionExpression simpleExpression = new SimpleAlarmConditionExpression(); AlarmConditionFilter filter = new AlarmConditionFilter(); filter.setArgument(argument); + filter.setValueType(EntityKeyValueType.STRING); StringFilterPredicate predicate = new StringFilterPredicate(); predicate.setOperation(stringOperation); predicate.setValue(conditionValue); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java index e9785d675b..6a1a36cf35 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/rule/condition/expression/AlarmConditionFilter.java @@ -20,6 +20,7 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.alarm.rule.condition.expression.predicate.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import java.io.Serializable; @@ -28,6 +29,8 @@ public class AlarmConditionFilter implements Serializable { @NotBlank private String argument; + @NotNull + private EntityKeyValueType valueType; @Valid @NotNull private KeyFilterPredicate predicate; From aab06c465d32a462fd340e349c42f8afbb5ce6ea Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 16:48:34 +0300 Subject: [PATCH 079/122] UI: Add metrics config to RELATED_ENTITIES_AGGREGATION --- .../calculated-field-output.component.html | 25 +- ...culated-field-metrics-panel.component.html | 168 ++++++++++++ ...culated-field-metrics-panel.component.scss | 31 +++ ...alculated-field-metrics-panel.component.ts | 176 +++++++++++++ ...culated-field-metrics-table.component.html | 111 ++++++++ ...culated-field-metrics-table.component.scss | 76 ++++++ ...alculated-field-metrics-table.component.ts | 242 ++++++++++++++++++ ...ities-aggregation-component.component.html | 7 +- ...ntities-aggregation-component.component.ts | 5 +- ...d-entities-aggregation-component.module.ts | 8 + .../shared/models/calculated-field.models.ts | 49 ++++ .../assets/locale/locale.constant-en_US.json | 30 ++- 12 files changed, 909 insertions(+), 19 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html index fcc000bf98..1faeb6adf6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html @@ -44,13 +44,7 @@ @if (simpleMode) { @if (hiddenName) {
- - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} @else { @@ -77,15 +71,18 @@ } - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} }
+ + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html new file mode 100644 index 0000000000..5c2985321a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -0,0 +1,168 @@ + +
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} + } + + +
+ +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }} +
+ + + + + +
{{ 'api-usage.tbel' | translate }} +
+ + + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss new file mode 100644 index 0000000000..ae692b6f93 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../../../scss/constants'; + +$panel-width: 520px; + +:host { + display: flex; + width: $panel-width; + max-width: 100%; + max-height: 80vh; + + .fixed-title-width { + @media #{$mat-xs} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts new file mode 100644 index 0000000000..6fb296a161 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts @@ -0,0 +1,176 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, output } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { FormBuilder, FormControl, ValidatorFn, Validators } from '@angular/forms'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { + AggFunction, + AggFunctionTranslations, + AggInputType, + AggInputTypeTranslations, + CalculatedFieldAggMetricValue +} from '@shared/models/calculated-field.models'; +import { delay, map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { merge, Observable, of } from 'rxjs'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +interface CalculatedFieldAggMetricValuePanel extends CalculatedFieldAggMetricValue { + allowFilter: boolean; +} + +@Component({ + selector: 'tb-calculated-field-metrics-panel', + templateUrl: './calculated-field-metrics-panel.component.html', + styleUrls: ['./calculated-field-metrics-panel.component.scss'] +}) +export class CalculatedFieldMetricsPanelComponent implements OnInit { + + @Input() buttonTitle: string; + @Input() metric: CalculatedFieldAggMetricValue; + @Input() usedNames: string[]; + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + metricDataApplied = output(); + filterExpanded = false; + functionArgs: Array + + metricForm = this.fb.group({ + name: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + function: [AggFunction.AVG], + allowFilter: [false], + filter: ['', Validators.required], + input: this.fb.group({ + type: [AggInputType.key], + key: ['', Validators.required], + function: ['', Validators.required], + }) + }); + + entityFilter: EntityFilter; + + readonly AggFunctions = Object.values(AggFunction) as AggFunction[]; + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly ScriptLanguage = ScriptLanguage; + readonly AggInputType = AggInputType; + readonly AggInputTypes = Object.values(AggInputType) as AggInputType[]; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + + constructor( + private fb: FormBuilder, + private popover: TbPopoverComponent + ) { + this.observeUpdatePosition(); + this.observeFilterAllowChange(); + this.observeInputTypeChange(); + } + + ngOnInit(): void { + const data: CalculatedFieldAggMetricValuePanel = { + ...this.metric, + allowFilter: !!this.metric.filter, + } + this.metricForm.patchValue(data, {emitEvent: false}); + + this.validateFilter(data.allowFilter); + this.validateInputTypeFilter(data.input?.type ?? AggInputType.key); + + this.functionArgs = ['ctx', ...this.arguments]; + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(this.arguments).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private observeFilterAllowChange(): void { + this.metricForm.get('allowFilter').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateFilter(value)); + } + + private observeInputTypeChange(): void { + this.metricForm.get('input.type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateInputTypeFilter(value)); + } + + private validateFilter(allowFilter = false): void { + if (allowFilter) { + this.metricForm.get('filter').enable({emitEvent: false}); + } else { + this.metricForm.get('filter').disable({emitEvent: false}); + } + this.filterExpanded = allowFilter; + } + + private validateInputTypeFilter(value: AggInputType): void { + const inputForm = this.metricForm.get('input'); + if (value === AggInputType.key) { + inputForm.get('key').enable({emitEvent: false}); + inputForm.get('function').disable({emitEvent: false}); + } else { + inputForm.get('key').disable({emitEvent: false}); + inputForm.get('function').enable({emitEvent: false}); + } + } + + saveZone(): void { + const value = this.metricForm.value as CalculatedFieldAggMetricValuePanel; + if (!value.allowFilter) { + delete value.filter; + } + delete value.allowFilter; + this.metricDataApplied.emit(value); + } + + cancel(): void { + this.popover.hide(); + } + + private uniqNameRequired(): ValidatorFn { + return (control: FormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.usedNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + + private forbiddenNameValidator(): ValidatorFn { + return (control: FormControl) => { + const trimmedValue = control.value.trim().toLowerCase(); + const forbiddenNames = ['ctx', 'e', 'pi']; + return forbiddenNames.includes(trimmedValue) ? { forbiddenName: true } : null; + }; + } + + private observeUpdatePosition(): void { + merge( + this.metricForm.get('allowFilter').valueChanges, + this.metricForm.get('input.type').valueChanges + ) + .pipe(delay(50), takeUntilDestroyed()) + .subscribe(() => this.popover.updatePosition()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html new file mode 100644 index 0000000000..481ad9ee13 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html @@ -0,0 +1,111 @@ + +
+
+ + + +
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + +
+
{{ metric.name }}
+ +
+ + + + + {{ 'calculated-fields.metrics.aggregation' | translate }} + + +
{{ AggFunctionTranslations.get(metric.function) | translate }}
+ + + + + {{ 'calculated-fields.metrics.filtered' | translate }} + + + {{ metric.filter ? 'check_box' : 'check_box_outline_blank' }} + + + + + {{ 'calculated-fields.metrics.value-source' | translate }} + + +
{{ AggInputTypeTranslations.get(metric.input.type) | translate }}
+ + + + + + +
+ + +
+ + + + +
+
+ {{ 'calculated-fields.metrics.no-metrics-configured' | translate }} +
+ @if (errorText) { + + } +
+
+ + @if (maxArgumentsPerCF && metricsFormArray.length >= maxArgumentsPerCF) { +
+ warning + {{ 'calculated-fields.metrics.max-metrics' | translate }} +
+ } +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss new file mode 100644 index 0000000000..430958d0f4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .arguments-table { + min-height: 108px; + + &-with-error { + min-height: 150px; + } + + .mat-mdc-table { + table-layout: fixed; + } + + .key-text { + font-size: 13px; + } + + .copy-argument-name { + visibility: hidden; + transition: visibility 0.1s; + } + + .argument-name-cell:hover { + .copy-argument-name { + visibility: visible; + } + } + } + + .max-args-warning { + .mat-icon { + color: #FAA405; + } + } + + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + +:host ::ng-deep { + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; + } + } + + .copy-argument-name { + .mat-icon { + font-size: 16px; + padding: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts new file mode 100644 index 0000000000..2e4ac30165 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts @@ -0,0 +1,242 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + Input, + Renderer2, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, +} from '@angular/forms'; +import { + AggFunctionTranslations, + AggInputTypeTranslations, + CalculatedFieldAggMetric, + CalculatedFieldAggMetricValue, +} from '@shared/models/calculated-field.models'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { isDefinedAndNotNull, isEqual } from '@core/utils'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { MatSort, SortDirection } from '@angular/material/sort'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-calculated-field-metrics-table', + templateUrl: './calculated-field-metrics-table.component.html', + styleUrls: [`calculated-field-metrics-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldMetricsTableComponent implements ControlValueAccessor, Validator, AfterViewInit { + + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + @ViewChild(MatSort, { static: true }) sort: MatSort; + + errorText = ''; + metricsFormArray = this.fb.array([]); + sortOrder = { direction: 'asc' as SortDirection, property: '' }; + dataSource = new CalculatedFieldMetricsDatasource(); + + displayColumns = ['name', 'function', 'filter', 'valueSource', 'actions'] + + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + readonly maxArgumentsPerCF = getCurrentAuthState(this.store).maxArgumentsPerCF - 2; + + private popoverComponent: TbPopoverComponent; + private propagateChange: (zonesObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private destroyRef: DestroyRef, + private store: Store + ) { + this.metricsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateDataSource(value); + this.propagateChange(this.getMetricsObject(value)); + }); + } + + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.metricsFormArray.value); + }); + } + + registerOnChange(fn: (zonesObj: Record) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any): void {} + + validate(): ValidationErrors | null { + this.updateErrorText(); + return this.errorText ? { metricsFormArray: false } : null; + } + + onDelete($event: Event, metric: CalculatedFieldAggMetricValue): void { + $event.stopPropagation(); + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + this.metricsFormArray.removeAt(index); + this.metricsFormArray.markAsDirty(); + } + + manageMetrics($event: Event, matButton: MatButton, metric = {} as CalculatedFieldAggMetricValue): void { + $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + const isExists = index !== -1; + const ctx = { + index, + metric, + buttonTitle: isExists ? 'action.apply' : 'action.add', + usedNames: this.metricsFormArray.value.map(({ name }) => name).filter(name => name !== metric.name), + arguments: this.arguments, + editorCompleter: this.editorCompleter, + highlightRules: this.highlightRules + }; + this.popoverComponent = this.popoverService.displayPopover({ + trigger, + renderer: this.renderer, + componentType: CalculatedFieldMetricsPanelComponent, + hostView: this.viewContainerRef, + preferredPlacement: isExists ? ['leftOnly', 'leftTopOnly', 'leftBottomOnly'] : ['rightOnly', 'rightTopOnly', 'rightBottomOnly'], + context: ctx, + isModal: true + }); + this.popoverComponent.tbComponentRef.instance.metricDataApplied.subscribe((value) => { + this.popoverComponent.hide(); + if (isExists) { + this.metricsFormArray.at(index).setValue(value); + } else { + this.metricsFormArray.push(this.fb.control(value)); + } + this.cd.markForCheck(); + }); + } + } + + private updateDataSource(value: CalculatedFieldAggMetricValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + + private updateErrorText(): void { + if (!this.metricsFormArray.controls.length) { + this.errorText = 'calculated-fields.metrics.metrics-empty'; + } else { + this.errorText = ''; + } + } + + private getMetricsObject(value: CalculatedFieldAggMetricValue[]): Record { + return value.reduce((acc, metricValue) => { + const { name, ...metric } = metricValue; + acc[name] = metric; + return acc; + }, {} as Record); + } + + writeValue(metrics: Record): void { + this.metricsFormArray.clear(); + this.populateZonesFormArray(metrics); + } + + private populateZonesFormArray(metrics: Record): void { + Object.keys(metrics).forEach(key => { + const value: CalculatedFieldAggMetricValue = { + ...metrics[key], + name: key + }; + this.metricsFormArray.push(this.fb.control(value), { emitEvent: false }); + }); + this.metricsFormArray.updateValueAndValidity(); + } + + private getSortValue(metric: CalculatedFieldAggMetricValue, column: string): string { + switch (column) { + case 'function': + return metric.function; + case 'filter': + return isDefinedAndNotNull(metric.filter).toString(); + default: + return metric.name; + } + } + + private sortData(data: CalculatedFieldAggMetricValue[]): CalculatedFieldAggMetricValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); + }); + } +} + +class CalculatedFieldMetricsDatasource extends TbTableDatasource { + constructor() { + super(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html index 1988141c2a..ce2d22fbbc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -51,8 +51,13 @@
- {{ 'calculated-fields.metrics' | translate }} + {{ 'calculated-fields.metrics.metrics' | translate }}
+ ({ scope: AttributeScope.SERVER_SCOPE, @@ -93,8 +94,8 @@ export class RelatedEntitiesAggregationComponentComponent implements ControlValu readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; - functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( - map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + arguments$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => Object.keys(argumentsObj)) ); argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts index a2c48c1896..b272f1e965 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -26,6 +26,12 @@ import { import { RelatedEntitiesAggregationComponentComponent } from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; +import { + CalculatedFieldMetricsTableComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; @NgModule({ imports: [ @@ -36,6 +42,8 @@ import { ], declarations: [ RelatedEntitiesAggregationComponentComponent, + CalculatedFieldMetricsTableComponent, + CalculatedFieldMetricsPanelComponent ], exports: [ RelatedEntitiesAggregationComponentComponent, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8ea6659bca..377b443fb6 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -113,6 +113,7 @@ export interface CalculatedFieldRelatedAggregationConfiguration { type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; relation: RelationPathLevel; arguments: Record; + metrics: Record; deduplicationIntervalInSec: number; useLatestTs: boolean; output: Omit; @@ -251,6 +252,54 @@ export interface CalculatedFieldArgument { timeWindow?: number; } +export enum AggFunction { + AVG='AVG', + MIN='MIN', + MAX='MAX', + SUM='SUM', + COUNT='COUNT', + COUNT_UNIQUE='COUNT_UNIQUE' +} + +export const AggFunctionTranslations = new Map([ + [AggFunction.AVG, 'calculated-fields.metrics.aggregation-type.avg'], + [AggFunction.MIN, 'calculated-fields.metrics.aggregation-type.min'], + [AggFunction.MAX, 'calculated-fields.metrics.aggregation-type.max'], + [AggFunction.SUM, 'calculated-fields.metrics.aggregation-type.sum'], + [AggFunction.COUNT, 'calculated-fields.metrics.aggregation-type.count'], + [AggFunction.COUNT_UNIQUE, 'calculated-fields.metrics.aggregation-type.count-unique'], +]) + +export interface CalculatedFieldAggMetric { + function: AggFunction; + filter?: string; + input: AggKeyInput | AggFunctionInput; +} + +export interface CalculatedFieldAggMetricValue extends CalculatedFieldAggMetric { + name: string; +} + +export enum AggInputType { + key = 'key', + function = 'function' +} + +export const AggInputTypeTranslations = new Map([ + [AggInputType.key, 'calculated-fields.metrics.value-source-type.key'], + [AggInputType.function, 'calculated-fields.metrics.value-source-type.function'], +]) + +export interface AggKeyInput { + type: AggInputType.key; + key: string; +} + +export interface AggFunctionInput { + type: AggInputType.function; + function: string; +} + export interface CalculatedFieldGeofencing { perimeterKeyName: string; reportStrategy: GeofencingReportStrategy; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0333c5ed2e..15207999f6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1159,10 +1159,36 @@ "output-key": "Output key", "copy-output-key": "Copy output key", "aggregation-path-related-entities": "Aggregation path to related entities", - "metrics": "Metrics", "deduplication-interval": "Deduplication interval", "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", "deduplication-interval-required": "Deduplication interval is required.", + "metrics": { + "metrics": "Metrics", + "metrics-empty": "At least one metric must be configured.", + "metric-name": "Metric name", + "copy-metric-name": "Copy metric name", + "aggregation": "Aggregation", + "aggregation-type": { + "avg": "Average", + "min": "Minimum", + "max": "Maximum", + "sum": "Sum", + "count": "Count", + "count-unique": "Count unique" + }, + "filtered": "Filtered", + "value-source": "Value source", + "value-source-type": { + "key": "Key", + "function": "Function" + }, + "no-metrics-configured": "No metrics configured", + "add-metric": "Add metric", + "max-metrics": "Maximum number of metrics reached.", + "metric-settings": "Metric settings", + "filter": "Filter", + "filter-hint": "Enables filtering of entities during aggregation. The filter function must return a boolean value and can use all configured arguments." + }, "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1182,7 +1208,7 @@ "output-key-max-length": "Output key should be less than 256 characters.", "output-key-forbidden": "Output key is reserved and cannot be used.", "entity-type-required": "Entity type is required", - "name-required": "Mame is required.", + "name-required": "Name is required.", "name-pattern": "Name is invalid.", "name-duplicate": "Name with such name already exists.", "name-max-length": "Name should be less than 256 characters.", From 0c68754160075282b455deb1c48f54290988fb44 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Oct 2025 17:12:05 +0300 Subject: [PATCH 080/122] changed propagation cf config to use relationPathLevel --- .../server/cf/CalculatedFieldIntegrationTest.java | 6 ++---- .../controller/CalculatedFieldControllerTest.java | 3 +-- .../state/PropagationCalculatedFieldStateTest.java | 4 ++-- .../PropagationCalculatedFieldConfiguration.java | 9 +++------ .../PropagationCalculatedFieldConfigurationTest.java | 12 ++++++++++-- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 209a2da6f1..8cf9c307a6 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -1025,8 +1025,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes cf.setConfigurationVersion(1); PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); - cfg.setDirection(EntitySearchDirection.TO); - cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setApplyExpressionToResolvedArguments(true); Argument arg = new Argument(); @@ -1105,8 +1104,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes cf.setConfigurationVersion(1); PropagationCalculatedFieldConfiguration cfg = new PropagationCalculatedFieldConfiguration(); - cfg.setDirection(EntitySearchDirection.TO); - cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode Argument arg = new Argument(); diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index aa3e802f70..4ebace6ae7 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -271,8 +271,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedFieldConfiguration getPropagationCalculatedFieldConfig(Map arguments) { var config = new PropagationCalculatedFieldConfiguration(); - config.setRelationType(EntityRelation.CONTAINS_TYPE); - config.setDirection(EntitySearchDirection.TO); + config.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); config.setApplyExpressionToResolvedArguments(false); config.setExpression(null); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java index 04a7ab5203..4f8b14f9b5 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.stats.DefaultStatsFactory; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; @@ -222,8 +223,7 @@ public class PropagationCalculatedFieldStateTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(boolean applyExpressionToResolvedArguments) { var config = new PropagationCalculatedFieldConfiguration(); - config.setDirection(EntitySearchDirection.TO); - config.setRelationType(EntityRelation.CONTAINS_TYPE); + config.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); config.setApplyExpressionToResolvedArguments(applyExpressionToResolvedArguments); Argument temperatureArg = new Argument(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index 5e8c822d78..1dfa9d3cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -15,13 +15,11 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; import java.util.List; @@ -33,9 +31,7 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField public static final String PROPAGATION_CONFIG_ARGUMENT = "propagationCtx"; @NotNull - private EntitySearchDirection direction; - @NotBlank - private String relationType; + private RelationPathLevel relation; private boolean applyExpressionToResolvedArguments; @@ -46,6 +42,7 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField @Override public void validate() { + relation.validate(); baseCalculatedFieldRestriction(); propagationRestriction(); if (!applyExpressionToResolvedArguments) { @@ -77,7 +74,7 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField public Argument toPropagationArgument() { var refDynamicSourceConfiguration = new RelationPathQueryDynamicSourceConfiguration(); - refDynamicSourceConfiguration.setLevels(List.of(new RelationPathLevel(direction, relationType))); + refDynamicSourceConfiguration.setLevels(List.of(relation)); var propagationArgument = new Argument(); propagationArgument.setRefDynamicSourceConfiguration(refDynamicSourceConfiguration); return propagationArgument; diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java index 36f63feed7..a3140ee63a 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import java.util.Map; import java.util.UUID; @@ -42,6 +43,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenConfigurationDisallowArgumentsWithReferencedEntity() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); Argument argumentWithRefEntityIdSet = new Argument(); argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("bda14084-f40e-4acc-9b85-9d1dd209bb64"))); cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); @@ -53,6 +55,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenConfigurationDisallowArgumentsWithDynamicReferenceConfiguration() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); Argument argumentWithDynamicRefEntitySource = new Argument(); argumentWithDynamicRefEntitySource.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); cfg.setArguments(Map.of("argumentWithDynamicRefEntitySource", argumentWithDynamicRefEntitySource)); @@ -64,6 +67,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenConfigurationHasNoArgumentsWithCurrentEntitySource() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); Argument argumentWithRefEntityIdSet = new Argument(); argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("3703e895-3f9b-4b75-a715-b68f1ad51944"))); cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); @@ -77,6 +81,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenUsedReservedPropagationArgumentName() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setArguments(Map.of(PROPAGATION_CONFIG_ARGUMENT, new Argument())); assertThatThrownBy(cfg::validate) .isInstanceOf(IllegalArgumentException.class) @@ -86,6 +91,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenUsedReservedCtxArgumentName() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setArguments(Map.of("ctx", new Argument())); assertThatThrownBy(cfg::validate) .isInstanceOf(IllegalArgumentException.class) @@ -95,6 +101,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenReferencedEntityKeyIsNotSet() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); Argument argument = new Argument(); cfg.setArguments(Map.of("someArgumentName", argument)); assertThatThrownBy(cfg::validate) @@ -105,6 +112,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenReferencedEntityKeyTypeIsTsRolling() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey("someKey", ArgumentType.TS_ROLLING, null); Argument argument = new Argument(); argument.setRefEntityKey(referencedEntityKey); @@ -118,6 +126,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateShouldThrowWhenExpressionIsNotSet() { var cfg = new PropagationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setArguments(Map.of("someArgumentName", new Argument())); cfg.setApplyExpressionToResolvedArguments(true); assertThatThrownBy(cfg::validate) @@ -128,8 +137,7 @@ public class PropagationCalculatedFieldConfigurationTest { @Test void validateToPropagationArgumentMethodCallReturnCorrectArgument() { var cfg = new PropagationCalculatedFieldConfiguration(); - cfg.setDirection(EntitySearchDirection.TO); - cfg.setRelationType(EntityRelation.CONTAINS_TYPE); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); Argument propagationArgument = cfg.toPropagationArgument(); assertThat(propagationArgument).isNotNull(); From 522369383b94d2b6b46cd3c377c6e28bca83022b Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 19:31:06 +0300 Subject: [PATCH 081/122] UI: Improvement popover in cf; Simplify style --- .../calculated-fields-table-config.ts | 4 +- ...ulated-field-argument-panel.component.html | 8 +- ...ulated-field-argument-panel.component.scss | 19 +- ...lculated-field-argument-panel.component.ts | 19 +- .../common/calculated-field-panel.scss | 58 +++ ...eofencing-zone-groups-panel.component.html | 421 +++++++++--------- ...eofencing-zone-groups-panel.component.scss | 20 +- ...-geofencing-zone-groups-panel.component.ts | 33 +- ...eofencing-zone-groups-table.component.html | 2 +- ...eofencing-zone-groups-table.component.scss | 76 ---- ...-geofencing-zone-groups-table.component.ts | 6 +- .../calculated-field-output.component.html | 2 +- ...culated-field-metrics-panel.component.html | 220 ++++----- ...alculated-field-metrics-panel.component.ts | 16 +- ...culated-field-metrics-table.component.html | 20 +- ...culated-field-metrics-table.component.scss | 76 ---- ...alculated-field-metrics-table.component.ts | 4 +- ...ities-aggregation-component.component.html | 4 +- ...ties-aggregation-component.component.scss} | 21 +- ...ntities-aggregation-component.component.ts | 1 + 20 files changed, 431 insertions(+), 599 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/common/calculated-field-panel.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-table.component.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss rename ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/{calculated-field-metrics-panel.component.scss => related-entities-aggregation-component.component.scss} (73%) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 4104b97221..1b48771247 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -111,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('expression', 'calculated-fields.expression', '300px'); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '250px'); expressionColumn.sortable = false; expressionColumn.cellContentFunction = entity => { const expressionLabel = this.getExpressionLabel(entity); @@ -124,7 +124,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('createdTime', 'common.created-time', this.datePipe, '150px')); this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); - this.columns.push(new EntityTableColumn('type', 'common.type', '80px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(new EntityTableColumn('type', 'common.type', '170px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)), () => ({whiteSpace: 'nowrap' }))); this.columns.push(expressionColumn); this.cellActionDescriptors.push( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 607b107094..1ffa7ccf77 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -15,9 +15,9 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-settings' | translate }}
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
@if (hint) {
{{ hint | translate }} @@ -190,7 +190,7 @@ }
-
+
-
-
-
{{ $index+1 }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} - } - - - - -
-
- -
-
- } -
- } @else { - {{ 'calculated-fields.no-level' | translate }} - } - @if (levelsFormArray().errors) { - - } -
-
- @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { -
- warning - {{ 'calculated-fields.max-allowed-levels-error' | translate }} -
- } @else { - - } -
- +
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
+
- - - @if (entityFilter.singleEntity?.id) { -
-
- {{ 'calculated-fields.perimeter-attribute-key' | translate }} + } + + +
+ + {{ 'calculated-fields.entity-zone-relationship' | translate }} +
+
+
+
calculated-fields.level
+
calculated-fields.direction-level
+
calculated-fields.relation-type
+
- @if (entityType === ArgumentEntityType.RelationQuery) { - - - @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { - - warning - - } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { - - warning - + @if (levelsFormArray()?.controls?.length) { +
+ @for (keyControl of levelsFormArray().controls; track trackByKey; ) { +
+
+ +
+
+
{{ $index + 1 }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} + } + + + + +
+
+ +
+
} - +
} @else { - + {{ 'calculated-fields.no-level' | translate }} + } + @if (levelsFormArray().errors) { + }
- } +
+ @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { +
+ warning + {{ 'calculated-fields.max-allowed-levels-error' | translate }} +
+ } @else { + + } +
+ +
+ + + @if (entityFilter.singleEntity?.id) {
-
{{ 'calculated-fields.report-strategy' | translate }}
- - - @for (strategy of GeofencingReportStrategyList; track strategy) { - {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} - } - - -
- -
- -
- {{ 'calculated-fields.create-relation-with-matched-zones' | translate }} +
+ {{ 'calculated-fields.perimeter-attribute-key' | translate }}
- -
-
{{ 'calculated-fields.direction' | translate }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionTranslations.get(direction) | translate }} + @if (entityType === ArgumentEntityType.RelationQuery) { + + + @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { + + warning + + } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { + + warning + } - - + + } @else { + + }
-
-
{{ 'calculated-fields.relation-type' | translate }}
- - + } +
+
{{ 'calculated-fields.report-strategy' | translate }}
+ + + @for (strategy of GeofencingReportStrategyList; track strategy) { + {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} + } + + +
+ +
+ +
+ {{ 'calculated-fields.create-relation-with-matched-zones' | translate }}
+ +
+
{{ 'calculated-fields.direction' | translate }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionTranslations.get(direction) | translate }} + } + + +
+
+
{{ 'calculated-fields.relation-type' | translate }}
+ +
-
+
@if (simpleMode) { @if (hiddenName) { -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html index 5c2985321a..b7c58f7633 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -15,23 +15,23 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.metrics.metric-settings' | translate }}
-
-
-
{{ 'calculated-fields.metrics.metric-name' | translate }}
- - - @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { +
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { - - warning - + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} } - -
-
-
{{ 'calculated-fields.metrics.aggregation' | translate }}
- - - @for (aggFunction of AggFunctions; track aggFunction) { - {{ AggFunctionTranslations.get(aggFunction) | translate }} - } - - -
+ + +
-
- - - - -
- {{ 'calculated-fields.metrics.filter' | translate }} -
- - - - - -
{{ 'api-usage.tbel' | translate }} +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }}
- - - -
- -
-
{{ 'calculated-fields.metrics.value-source' | translate }}
- - - @for (inputType of AggInputTypes; track inputType) { - {{ AggInputTypeTranslations.get(inputType) | translate }} - } - - -
- @if (this.metricForm.get('input.type').value === AggInputType.key) { -
-
{{ 'calculated-fields.argument-name' | translate }}
- - -
- } @else { + + + + {{ 'api-usage.tbel' | translate }}
- } - + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } +
-
+