From 91ff77d20bc1e445eca314077fe77a3065657f8f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:42:48 +0300 Subject: [PATCH 001/281] Git Check --- .../main/resources/sql/schema-entities.sql | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9c95f385f8..c8872cdc15 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,5 +1,5 @@ -- --- Copyright © 2016-2024 The Thingsboard Authors +-- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- -- Licensed under the Apache License, Version 2.0 (the "License"); -- you may not use this file except in compliance with the License. @@ -896,3 +896,31 @@ CREATE TABLE IF NOT EXISTS mobile_app_settings ( qr_code_config VARCHAR(100000), CONSTRAINT mobile_app_settings_tenant_id_unq_key UNIQUE (tenant_id) ); + +CREATE TABLE IF NOT EXISTS calculated_field ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, + type varchar(32) NOT NULL, + name varchar(255) NOT NULL, + configuration_version int DEFAULT 0, + configuration varchar(1000000), + version BIGINT DEFAULT 1, + external_id UUID, + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(10000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); From fd0af733c1eba5c65fcedf4c4225d87f1983cdc3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:59:08 +0300 Subject: [PATCH 002/281] LICENSE CHECK --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index c8f142f0bf..2717e8bf56 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ +12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From d8fd9234def7fa2dba80d9328e53f887e6aa952d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 30 Oct 2024 17:16:39 +0200 Subject: [PATCH 003/281] Revert "LICENSE CHECK" This reverts commit fd0af733c1eba5c65fcedf4c4225d87f1983cdc3. --- LICENSE | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE b/LICENSE index 2717e8bf56..c8f142f0bf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ -12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From 32bbd835419b00b2e4dd02e973b13d3b00291e53 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 31 Oct 2024 16:42:28 +0200 Subject: [PATCH 004/281] added calculated field entity, dao and controller --- .../server/controller/BaseController.java | 12 ++ .../controller/CalculatedFieldController.java | 95 ++++++++++++++ .../service/security/permission/Resource.java | 4 +- .../permission/TenantAdminPermissions.java | 1 + .../CalculatedFieldService.java | 31 +++++ .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedField.java | 94 ++++++++++++++ .../common/data/id/CalculatedFieldId.java | 44 +++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 75 +++++++++++ .../calculated_field/CalculatedFieldDao.java | 22 ++++ .../server/dao/model/ModelConstants.java | 14 +++ .../dao/model/sql/CalculatedFieldEntity.java | 118 ++++++++++++++++++ .../CalculatedFieldRepository.java | 24 ++++ .../JpaCalculatedFieldDao.java | 47 +++++++ .../main/resources/sql/schema-entities.sql | 47 ++++--- 17 files changed, 617 insertions(+), 17 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index da7d6243fc..10323f4fa0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,6 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -77,6 +78,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -124,6 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -343,6 +346,9 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -645,6 +651,8 @@ public abstract class BaseController { case MOBILE_APP: checkMobileAppId(new MobileAppId(entityId.getId()), operation); return; + case CALCULATED_FIELD: + checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -915,6 +923,10 @@ public abstract class BaseController { } } + protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + } + protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java new file mode 100644 index 0000000000..6bb5fc15f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldController extends BaseController{ + + private final CalculatedFieldService calculatedFieldService; + + public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", + notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + + "The newly created Calculated Field Id will be present in the response. " + + "Specify existing Calculated Field Id to update the calculated field. " + + "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) + @ResponseBody + public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") + @RequestBody CalculatedField calculatedField) throws Exception { + calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + return calculatedFieldService.save(calculatedField); + } + + @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", + notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) + @ResponseBody + public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + } + + + @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", + notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { + checkParameter(CALCULATED_FIELD_ID, strCalculatedField); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); + checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 16a3c4be59..7ec040ecd8 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -49,7 +49,9 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + private final Set entityTypes; Resource() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 10807e4b5a..897581cc91 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -51,6 +51,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java new file mode 100644 index 0000000000..7f79f481b0 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; + +public interface CalculatedFieldService extends EntityDaoService { + + CalculatedField save(CalculatedField calculatedField); + + CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index b5ce79e20f..7b6e99095f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -60,7 +60,8 @@ public enum EntityType { QUEUE_STATS(34), OAUTH2_CLIENT(35), DOMAIN(36), - MOBILE_APP(37); + MOBILE_APP(37), + CALCULATED_FIELD(38); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java new file mode 100644 index 0000000000..d780262726 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { + + private static final long serialVersionUID = 4491966747773381420L; + + private TenantId tenantId; + private EntityId entityId; + + @NoXss + @Length(fieldName = "type") + private String type; + @NoXss + @Length(fieldName = "name") + @Schema(description = "User defined name of the calculated field.") + private String name; + @Schema(description = "Version of calculated field configuration.", example = "0") + private int configurationVersion; + @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + @Getter + @Setter + private Long version; + @Getter + @Setter + private CalculatedFieldId externalId; + + public CalculatedField() { + super(); + } + + public CalculatedField(CalculatedFieldId id) { + super(id); + } + + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { + super(); + this.tenantId = tenantId; + this.entityId = entityId; + this.type = type; + this.name = name; + this.configurationVersion = configurationVersion; + this.configuration = configuration; + this.version = version; + this.externalId = externalId; + } + + @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") + @Override + public CalculatedFieldId getId() { + return super.getId(); + } + + @Schema(description = "Timestamp of the calculated field creation, in milliseconds", example = "1609459200000", accessMode = Schema.AccessMode.READ_ONLY) + @Override + public long getCreatedTime() { + return super.getCreatedTime(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java new file mode 100644 index 0000000000..0de6209b9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldId fromString(String calculatedFieldId) { + return new CalculatedFieldId(UUID.fromString(calculatedFieldId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD", allowableValues = "CALCULATED_FIELD") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 5a85e6ce67..2b53011bdd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -111,6 +111,8 @@ public class EntityIdFactory { return new MobileAppId(uuid); case DOMAIN: return new DomainId(uuid); + case CALCULATED_FIELD: + return new CalculatedFieldId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f84c7f7c9b..985e7518f5 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -58,6 +58,7 @@ enum EntityTypeProto { OAUTH2_CLIENT = 35; DOMAIN = 36; MOBILE_APP = 37; + CALCULATED_FIELD = 38; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java new file mode 100644 index 0000000000..f9f7681a3a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service("CalculatedFieldDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseCalculatedFieldService implements CalculatedFieldService { + + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; + + private final CalculatedFieldDao calculatedFieldDao; + + + @Override + public CalculatedField save(CalculatedField calculatedField) { + log.trace("Executing save, [{}]", calculatedField); + return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + } + + @Override + public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); + } + + @Override + public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java new file mode 100644 index 0000000000..71decef684 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.Dao; + +public interface CalculatedFieldDao extends Dao { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f85cd4bac4..c37cbfca8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -691,6 +691,20 @@ public class ModelConstants { public static final String MOBILE_APP_SETTINGS_IOS_CONFIG_PROPERTY = "ios_config"; public static final String MOBILE_APP_SETTINGS_QR_CODE_CONFIG_PROPERTY = "qr_code_config"; + /** + * Calculated fields constants. + */ + public static final String CALCULATED_FIELD_TABLE_NAME = "calculated_field"; + public static final String CALCULATED_FIELD_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_TYPE = "type"; + public static final String CALCULATED_FIELD_NAME = "name"; + public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; + public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; + public static final String CALCULATED_FIELD_VERSION = "version"; + public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java new file mode 100644 index 0000000000..35c50c561f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_TABLE_NAME) +public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_TYPE) + private String type; + + @Column(name = CALCULATED_FIELD_NAME) + private String name; + + @Column(name = CALCULATED_FIELD_CONFIGURATION_VERSION) + private int configurationVersion; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_CONFIGURATION) + private JsonNode configuration; + + @Column(name = CALCULATED_FIELD_VERSION) + private Long version; + + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) + private UUID externalId; + + public CalculatedFieldEntity() { + super(); + } + + public CalculatedFieldEntity(CalculatedField calculatedField) { + this.setUuid(calculatedField.getUuidId()); + this.createdTime = calculatedField.getCreatedTime(); + this.tenantId = calculatedField.getTenantId().getId(); + this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityId = calculatedField.getEntityId().getId(); + this.type = calculatedField.getType(); + this.name = calculatedField.getName(); + this.configurationVersion = calculatedField.getConfigurationVersion(); + this.configuration = calculatedField.getConfiguration(); + this.version = calculatedField.getVersion(); + if (calculatedField.getExternalId() != null) { + this.externalId = calculatedField.getExternalId().getId(); + } + } + + @Override + public CalculatedField toData() { + CalculatedField calculatedField = new CalculatedField(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(configuration); + calculatedField.setVersion(version); + if (externalId != null) { + calculatedField.setExternalId(new CalculatedFieldId(externalId)); + } + return calculatedField; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java new file mode 100644 index 0000000000..79146cf317 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; + +import java.util.UUID; + +public interface CalculatedFieldRepository extends JpaRepository { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java new file mode 100644 index 0000000000..33cf926eaf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { + + private final CalculatedFieldRepository calculatedFieldRepository; + + @Override + protected Class getEntityClass() { + return CalculatedFieldEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldRepository; + } +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a26bb03e62..545486180d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,3 +1,19 @@ +-- +-- Copyright © 2016-2024 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + -- -- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- @@ -891,6 +907,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, created_time bigint NOT NULL, tenant_id uuid NOT NULL, + entity_type VARCHAR(32), entity_id uuid NOT NULL, type varchar(32) NOT NULL, name varchar(255) NOT NULL, @@ -899,18 +916,18 @@ CREATE TABLE IF NOT EXISTS calculated_field ( version BIGINT DEFAULT 1, external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) -); - -CREATE TABLE IF NOT EXISTS calculated_field_link ( - id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - entity_id uuid NOT NULL, --- target_id uuid NOT NULL, - calculated_field_id uuid NOT NULL, - configuration varchar(10000), - CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), - CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE -); + CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +-- CREATE TABLE IF NOT EXISTS calculated_field_link ( +-- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, +-- created_time bigint NOT NULL, +-- tenant_id uuid NOT NULL, +-- entity_id uuid NOT NULL, +-- -- target_id uuid NOT NULL, +-- calculated_field_id uuid NOT NULL, +-- configuration varchar(10000), +-- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), +-- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), +-- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +-- ); From e2d17a822d7519e0d611d06b1ec1abd2160e812c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 1 Nov 2024 13:34:04 +0200 Subject: [PATCH 005/281] added tests --- .../server/controller/BaseController.java | 1 + .../CalculatedFieldControllerTest.java | 143 ++++++++++++++++++ .../calculated_field/CalculatedField.java | 24 ++- .../BaseCalculatedFieldService.java | 1 - .../CalculatedFieldServiceTest.java | 123 +++++++++++++++ 5 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 10323f4fa0..6d52bdb358 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -653,6 +653,7 @@ public abstract class BaseController { return; case CALCULATED_FIELD: checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java new file mode 100644 index 0000000000..05ffd03970 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldControllerTest extends AbstractControllerTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); + + private Tenant savedTenant; + private User tenantAdmin; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testSaveCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testGetCalculatedFieldById() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); + + assertThat(fetchedCalculatedField).isNotNull(); + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testDeleteCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); + + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index d780262726..0bd8ded8f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -21,10 +21,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,7 +31,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { private static final long serialVersionUID = 4491966747773381420L; @@ -91,4 +88,21 @@ public class CalculatedField extends BaseData implements HasN return super.getCreatedTime(); } + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedField[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", type='").append(type) + .append(", name='").append(name) + .append(", configurationVersion=").append(configurationVersion) + .append(", configuration=").append(configuration) + .append(", version=").append(version) + .append(", externalId=").append(externalId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index f9f7681a3a..530a533085 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -39,7 +39,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; - @Override public CalculatedField save(CalculatedField calculatedField) { log.trace("Executing save, [{}]", calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java new file mode 100644 index 0000000000..d2e3d40c5c --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldServiceTest extends AbstractServiceTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); + + @Autowired + private CalculatedFieldService calculatedFieldService; + + private ListeningExecutorService executor; + + @Before + public void before() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); + } + + @After + public void after() { + executor.shutdownNow(); + } + + @Test + public void testSaveCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(calculatedField.getTenantId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testFindCalculatedFieldById() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); + + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testDeleteCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + + assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} From 03ff7c17ac0b4e7049f9bb7fc060e0a219237a9f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 4 Nov 2024 15:57:37 +0200 Subject: [PATCH 006/281] added calculated field link entity and its dao --- .../controller/CalculatedFieldController.java | 4 +- .../service/security/permission/Resource.java | 3 +- .../CalculatedFieldControllerTest.java | 23 +++-- .../CalculatedFieldService.java | 3 + .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedFieldLink.java | 71 ++++++++++++++ .../common/data/id/CalculatedFieldLinkId.java | 45 +++++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 66 ++++++++++++- .../calculated_field/CalculatedFieldDao.java | 2 +- .../CalculatedFieldLinkDao.java | 27 ++++++ .../server/dao/model/ModelConstants.java | 10 ++ .../model/sql/CalculatedFieldLinkEntity.java | 92 +++++++++++++++++++ .../CalculatedFieldDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkRepository.java | 27 ++++++ .../JpaCalculatedFieldLinkDao.java | 54 +++++++++++ .../main/resources/sql/schema-entities.sql | 24 ++--- .../CalculatedFieldServiceTest.java | 90 ++++++++++++++++-- .../CalculatedFieldDataValidatorTest.java | 57 ++++++++++++ .../CalculatedFieldLinkDataValidatorTest.java | 57 ++++++++++++ 22 files changed, 702 insertions(+), 41 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 6bb5fc15f9..26a3539a46 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -44,7 +44,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @RequestMapping("/api") @RequiredArgsConstructor @Slf4j -public class CalculatedFieldController extends BaseController{ +public class CalculatedFieldController extends BaseController { private final CalculatedFieldService calculatedFieldService; @@ -61,7 +61,7 @@ public class CalculatedFieldController extends BaseController{ @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") - @RequestBody CalculatedField calculatedField) throws Exception { + @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 7ec040ecd8..f57f460fc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,8 @@ public enum Resource { NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + CALCULATED_FIELD(EntityType.CALCULATED_FIELD), + CALCULATED_FIELD_LINK(EntityType.CALCULATED_FIELD_LINK); private final Set entityTypes; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 05ffd03970..685a58d48c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -19,6 +19,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; @@ -26,18 +27,13 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.UUID; - import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class CalculatedFieldControllerTest extends AbstractControllerTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); - private Tenant savedTenant; - private User tenantAdmin; @Before public void beforeTest() throws Exception { @@ -48,14 +44,14 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { savedTenant = saveTenant(tenant); assertThat(savedTenant).isNotNull(); - tenantAdmin = new User(); + User tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); - tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + createUserAndLogin(tenantAdmin, "testPassword1"); } @After @@ -67,7 +63,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testSaveCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -93,7 +90,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testGetCalculatedFieldById() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); @@ -107,7 +105,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testDeleteCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -119,9 +118,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { } - private CalculatedField getCalculatedField() { + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index 7f79f481b0..e1779e1d33 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -28,4 +29,6 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 7b6e99095f..37d77bd2c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -61,7 +61,8 @@ public enum EntityType { OAUTH2_CLIENT(35), DOMAIN(36), MOBILE_APP(37), - CALCULATED_FIELD(38); + CALCULATED_FIELD(38), + CALCULATED_FIELD_LINK(39); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java new file mode 100644 index 0000000000..79f05c9b58 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldLink extends BaseData { + + private static final long serialVersionUID = 6492846246722091530L; + + private TenantId tenantId; + private EntityId entityId; + + @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) + private CalculatedFieldId calculatedFieldId; + @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + + public CalculatedFieldLink() { + super(); + } + + public CalculatedFieldLink(CalculatedFieldLinkId id) { + super(id); + } + + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.configuration = configuration; + this.calculatedFieldId = calculatedFieldId; + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedFieldLink[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", calculatedFieldId=").append(calculatedFieldId) + .append(", configuration=").append(configuration) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java new file mode 100644 index 0000000000..8817aa8a1e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldLinkId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldLinkId fromString(String calculatedFieldLinkId) { + return new CalculatedFieldLinkId(UUID.fromString(calculatedFieldLinkId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD_LINK", allowableValues = "CALCULATED_FIELD_LINK") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 2b53011bdd..973aa5202e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -113,6 +113,8 @@ public class EntityIdFactory { return new DomainId(uuid); case CALCULATED_FIELD: return new CalculatedFieldId(uuid); + case CALCULATED_FIELD_LINK: + return new CalculatedFieldLinkId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 985e7518f5..875a531566 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -59,6 +59,7 @@ enum EntityTypeProto { DOMAIN = 36; MOBILE_APP = 37; CALCULATED_FIELD = 38; + CALCULATED_FIELD_LINK = 39; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index 530a533085..e7bc3be314 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -20,13 +20,21 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.service.DataValidator; +import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; @Service("CalculatedFieldDaoService") @@ -38,11 +46,28 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; private final CalculatedFieldDao calculatedFieldDao; + private final CalculatedFieldLinkDao calculatedFieldLinkDao; + private final DeviceService deviceService; + private final AssetService assetService; + private final DataValidator calculatedFieldDataValidator; + private final DataValidator calculatedFieldLinkDataValidator; @Override public CalculatedField save(CalculatedField calculatedField) { - log.trace("Executing save, [{}]", calculatedField); - return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + try { + TenantId tenantId = calculatedField.getTenantId(); + checkEntityExistence(tenantId, calculatedField.getEntityId()); + log.trace("Executing save calculated field, [{}]", calculatedField); + CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); + createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + return savedCalculatedField; + } catch (Exception e) { + checkConstraintViolation(e, + "calculated_field_unq_key", "Calculated Field with such name is already in exists!", + "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + throw e; + } } @Override @@ -61,6 +86,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); + try { + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); + } catch (Exception e) { + checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); + throw e; + } + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -71,4 +108,29 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + } + } + + private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); + saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + } + + private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + return calculatedFieldLink; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index 71decef684..e4bd019b18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -18,5 +18,5 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.dao.Dao; -public interface CalculatedFieldDao extends Dao { +public interface CalculatedFieldDao extends Dao { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java new file mode 100644 index 0000000000..1c422311eb --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.Dao; + +import java.util.UUID; + +public interface CalculatedFieldLinkDao extends Dao { + + CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index c37cbfca8e..368478db6b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -705,6 +705,16 @@ public class ModelConstants { public static final String CALCULATED_FIELD_VERSION = "version"; public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + /** + * Calculated field links constants. + */ + public static final String CALCULATED_FIELD_LINK_TABLE_NAME = "calculated_field_link"; + public static final String CALCULATED_FIELD_LINK_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + public static final String CALCULATED_FIELD_LINK_CONFIGURATION = "configuration"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java new file mode 100644 index 0000000000..9f2efb230b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TENANT_ID_COLUMN; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_LINK_TABLE_NAME) +public class CalculatedFieldLinkEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_LINK_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID) + private UUID calculatedFieldId; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_LINK_CONFIGURATION) + private JsonNode configuration; + + + public CalculatedFieldLinkEntity() { + super(); + } + + public CalculatedFieldLinkEntity(CalculatedFieldLink calculatedFieldLink) { + this.setUuid(calculatedFieldLink.getUuidId()); + this.createdTime = calculatedFieldLink.getCreatedTime(); + this.tenantId = calculatedFieldLink.getTenantId().getId(); + this.entityType = calculatedFieldLink.getEntityId().getEntityType(); + this.entityId = calculatedFieldLink.getEntityId().getId(); + this.calculatedFieldId = calculatedFieldLink.getCalculatedFieldId().getId(); + this.configuration = calculatedFieldLink.getConfiguration(); + } + + @Override + public CalculatedFieldLink toData() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(TenantId.fromUUID(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(configuration); + return calculatedFieldLink; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java new file mode 100644 index 0000000000..e50f2c902a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldDao calculatedFieldDao; + + @Override + protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { + CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java new file mode 100644 index 0000000000..83c30d4d03 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldLinkDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldLinkDao calculatedFieldLinkDao; + + @Override + protected CalculatedFieldLink validateUpdate(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + CalculatedFieldLink old = calculatedFieldLinkDao.findById(calculatedFieldLink.getTenantId(), calculatedFieldLink.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field link!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java new file mode 100644 index 0000000000..1339cf4478 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; + +import java.util.UUID; + +public interface CalculatedFieldLinkRepository extends JpaRepository { + + CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java new file mode 100644 index 0000000000..0721e08d7f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { + + private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + + @Override + public CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId) { + return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndEntityId(tenantId, entityId)); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldLinkEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldLinkRepository; + } + +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 545486180d..790317fcb0 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -919,15 +919,15 @@ CREATE TABLE IF NOT EXISTS calculated_field ( CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) ); --- CREATE TABLE IF NOT EXISTS calculated_field_link ( --- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, --- created_time bigint NOT NULL, --- tenant_id uuid NOT NULL, --- entity_id uuid NOT NULL, --- -- target_id uuid NOT NULL, --- calculated_field_id uuid NOT NULL, --- configuration varchar(10000), --- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), --- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), --- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE --- ); +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_link_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(1000000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java index d2e3d40c5c..fdff4bc2f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -23,22 +23,28 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.AbstractServiceTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CalculatedFieldServiceTest extends AbstractServiceTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); - @Autowired private CalculatedFieldService calculatedFieldService; + @Autowired + private DeviceService deviceService; private ListeningExecutorService executor; @@ -54,7 +60,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -77,10 +84,42 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { } @Test - public void testFindCalculatedFieldById() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + public void testSaveCalculatesFieldWithNonExistingDeviceId() { + CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); + } + + @Test + public void testSaveCalculatedFieldWithExistingName() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such name is already in exists!"); + } + + @Test + public void testSaveCalculatedFieldWithExistingExternalId() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); + + calculatedFieldService.save(calculatedField); + + calculatedField.setName("Test 2"); + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such external id already exists!"); + } + @Test + public void testFindCalculatedFieldById() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); @@ -90,18 +129,33 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testDeleteCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + CalculatedField savedCalculatedField = saveValidCalculatedField(); calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); } - private CalculatedField getCalculatedField() { + @Test + public void testSaveCalculatedFieldLinkIfCalculatedFieldForSuchEntityExists() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + CalculatedFieldLink calculatedFieldLink = getCalculatedFieldLink(savedCalculatedField); + + assertThatThrownBy(() -> calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field for such entity id is already exists!"); + } + + private CalculatedField saveValidCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); @@ -120,4 +174,20 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } + private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + return calculatedFieldLink; + } + + private Device createTestDevice() { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test"); + return deviceService.saveDevice(device); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java new file mode 100644 index 0000000000..bc7990ca64 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldDataValidator.class) +public class CalculatedFieldDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("7b5229e9-166e-41a9-a257-3b1dafad1b04")); + private final CalculatedFieldId CALCULATED_FIELD_ID = new CalculatedFieldId(UUID.fromString("060fbe45-fbb2-4549-abf3-f72a6be3cb9f")); + + @MockBean + private CalculatedFieldDao calculatedFieldDao; + @SpyBean + private CalculatedFieldDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test"); + + given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java new file mode 100644 index 0000000000..a74c59d7ad --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldLinkDataValidator.class) +public class CalculatedFieldLinkDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("2ba09d99-6143-43dc-b645-381fc0c43ebe")); + private final CalculatedFieldLinkId CALCULATED_FIELD_LINK_ID = new CalculatedFieldLinkId(UUID.fromString("a5609ef4-cb42-43ce-9b23-e090a4878d1c")); + + @MockBean + private CalculatedFieldLinkDao calculatedFieldLinkDao; + @SpyBean + private CalculatedFieldLinkDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(CALCULATED_FIELD_LINK_ID); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(UUID.fromString("136477af-fd07-4498-b9c9-54fe50e82992"))); + + given(calculatedFieldLinkDao.findById(TENANT_ID, CALCULATED_FIELD_LINK_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field link!"); + } + +} \ No newline at end of file From ac20551bf41d43a5e094f394a9f1d889c36e0b02 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 5 Nov 2024 17:05:10 +0200 Subject: [PATCH 007/281] added support of housekeeper deletion --- .../controller/CalculatedFieldController.java | 13 +++- ...CalculatedFieldsDeletionTaskProcessor.java | 43 +++++++++++ .../permission/CustomerUserPermissions.java | 31 +++++++- .../security/permission/Operation.java | 2 +- .../permission/TenantAdminPermissions.java | 21 +++-- .../CalculatedFieldControllerTest.java | 1 - .../CalculatedFieldService.java | 5 ++ .../data/housekeeper/HousekeeperTask.java | 4 + .../data/housekeeper/HousekeeperTaskType.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 4 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 31 +++++++- .../calculated_field/CalculatedFieldDao.java | 9 +++ .../dao/device/DeviceProfileServiceImpl.java | 4 +- .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/entity/AbstractEntityService.java | 5 ++ .../dao/housekeeper/CleanUpService.java | 1 + .../CalculatedFieldRepository.java | 6 ++ .../JpaCalculatedFieldDao.java | 16 ++++ .../dao/service/AssetProfileServiceTest.java | 21 +++++ .../server/dao/service/AssetServiceTest.java | 77 ++++++++++++------- .../CalculatedFieldServiceTest.java | 5 +- .../dao/service/DeviceProfileServiceTest.java | 25 +++++- .../server/dao/service/DeviceServiceTest.java | 20 +++++ .../CalculatedFieldLinkDataValidatorTest.java | 2 +- 25 files changed, 300 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java rename dao/src/test/java/org/thingsboard/server/dao/{calculated_field => service}/CalculatedFieldServiceTest.java (97%) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 26a3539a46..b3f2c018cb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -30,11 +30,11 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -63,7 +63,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); - checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); } @@ -76,7 +76,10 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + checkNotNull(calculatedField); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + return calculatedField; } @@ -88,7 +91,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + TenantId tenantId = getTenantId(); + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java new file mode 100644 index 0000000000..18d84b02ce --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 46cd1c2979..0c6dbc1599 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityPermissionChecker); - put(Resource.DEVICE, customerEntityPermissionChecker); + put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,6 +85,29 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; + private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = + new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, + Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, + Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, + Operation.WRITE_CALCULATED_FIELD) { + + @Override + @SuppressWarnings("unchecked") + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + + if (!super.hasPermission(user, operation, entityId, entity)) { + return false; + } + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + if (!(entity instanceof HasCustomerId)) { + return false; + } + return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); + } + }; + private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { @@ -188,7 +211,8 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( + Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { @Override @SuppressWarnings("unchecked") @@ -202,4 +226,5 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; + } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 9465a91cc8..d029d78cff 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -19,6 +19,6 @@ public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, - ASSIGN_TO_TENANT + ASSIGN_TO_TENANT, READ_CALCULATED_FIELD, WRITE_CALCULATED_FIELD } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 897581cc91..c730637148 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,15 +23,15 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="tenantAdminPermissions") +@Component(value = "tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityPermissionChecker); - put(Resource.DEVICE, tenantEntityPermissionChecker); + put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -51,14 +51,23 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); - put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); + } + }; + + public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { + @Override + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 685a58d48c..d9f4ac0377 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -115,7 +115,6 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) .andExpect(status().isOk()); doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); - } private CalculatedField getCalculatedField(DeviceId deviceId) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index e1779e1d33..f271f4ed24 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -29,6 +30,10 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 8757b1121e..7a87528553 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -81,6 +81,10 @@ public class HousekeeperTask implements Serializable { return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } + public static HousekeeperTask deleteCalculatedFields(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index edd13b3d5d..236844c17d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -30,7 +30,8 @@ public enum HousekeeperTaskType { DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), - DELETE_ENTITIES("entities deletion"); + DELETE_ENTITIES("entities deletion"), + DELETE_CALCULATED_FIELDS("calculated fields deletion"); private final String description; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7d4d5300f9..baf20219ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -72,7 +79,7 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing findById, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); @@ -80,12 +87,21 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(entityId.getId(), id -> "Incorrect entityId " + id); + List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); + return calculatedFields.size(); + } + @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); @@ -98,6 +114,11 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -114,6 +135,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index e4bd019b18..d0d0a3c386 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -16,7 +16,16 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldDao extends Dao { + + boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + + List removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 09e2be01e0..84dc1a7d14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService { + + boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 33cf926eaf..9a0eef26c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -15,16 +15,21 @@ */ package org.thingsboard.server.dao.sql.calculated_field; +import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -35,6 +40,17 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.removeAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override protected Class getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java index 7271a0fc8a..c11d8c6e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java @@ -28,10 +28,12 @@ import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; @@ -43,6 +45,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class AssetProfileServiceTest extends AbstractServiceTest { @@ -54,6 +57,8 @@ public class AssetProfileServiceTest extends AbstractServiceTest { AssetProfileService assetProfileService; @Autowired AssetService assetService; + @Autowired + private CalculatedFieldService calculatedFieldService; @Test public void testSaveAssetProfile() { @@ -380,5 +385,21 @@ public class AssetProfileServiceTest extends AbstractServiceTest { assertThat(assetProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteAssetProfileIfCalculatedFieldExists() { + AssetProfile assetProfile = this.createAssetProfile(tenantId, "Asset Profile"); + AssetProfile savedAssetProfile = assetProfileService.saveAssetProfile(assetProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAssetProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Asset Profile is prohibited!"); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 1976f173a5..e1e1bf70ed 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -47,6 +49,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @DaoSqlTest @@ -63,6 +66,8 @@ public class AssetServiceTest extends AbstractServiceTest { @Autowired private AssetProfileService assetProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; private IdComparator idComparator = new IdComparator<>(); @@ -214,24 +219,24 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); try { - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(assetService.saveAsset(asset)); } @@ -267,10 +272,10 @@ public class AssetServiceTest extends AbstractServiceTest { @Test public void testFindAssetsByTenantId() { List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(assetService.saveAsset(asset)); } @@ -303,11 +308,11 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetsByTenantIdAndName() { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -315,11 +320,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -381,11 +386,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -394,11 +399,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -464,10 +469,10 @@ public class AssetServiceTest extends AbstractServiceTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = assetService.saveAsset(asset); assets.add(new AssetInfo(assetService.assignAssetToCustomer(tenantId, asset.getId(), customerId), customer.getTitle(), customer.isPublic(), "default")); @@ -508,11 +513,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -521,11 +526,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -596,11 +601,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -610,11 +615,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -848,4 +853,24 @@ public class AssetServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteAssetIfCalculatedFieldExists() { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete asset that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java similarity index 97% rename from dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index fdff4bc2f6..46cc478904 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -28,10 +28,9 @@ import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.service.AbstractServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index aa14e20151..9bb1abc3c8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -49,6 +51,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; @DaoSqlTest @@ -60,6 +63,9 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; + @Autowired + private CalculatedFieldService calculatedFieldService; + private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -397,7 +403,7 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { var profileA = deviceProfileService.saveDeviceProfile( - createDeviceProfile(tenantId, "profile A")); + createDeviceProfile(tenantId, "profile A")); deviceProfiles.add(deviceProfileService.saveDeviceProfile(profileA)); @@ -478,4 +484,21 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteDeviceProfileIfCalculatedFieldExists() { + DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedDeviceProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Device Profile is prohibited!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 9fe3f85463..f2508090b2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -87,6 +89,8 @@ public class DeviceServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; @SpyBean private DeviceCredentialsDataValidator validator; @@ -1198,4 +1202,20 @@ public class DeviceServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteDeviceIfCalculatedFieldExists() { + Device device = saveDevice(tenantId, "Test"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete device that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index a74c59d7ad..c9fa57a751 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -54,4 +54,4 @@ public class CalculatedFieldLinkDataValidatorTest { .hasMessage("Can't update non existing calculated field link!"); } -} \ No newline at end of file +} From c223f1d2ba6754b739ba36f29f1468c4bea2d684 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:06:05 +0200 Subject: [PATCH 008/281] declined permissions for customers --- .../permission/CustomerUserPermissions.java | 27 ++----------------- .../permission/TenantAdminPermissions.java | 19 +++---------- .../dao/asset/AssetProfileServiceImpl.java | 4 +-- .../dao/device/DeviceProfileServiceImpl.java | 4 +-- .../dao/service/AssetProfileServiceTest.java | 22 --------------- .../dao/service/DeviceProfileServiceTest.java | 20 -------------- 6 files changed, 10 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 0c6dbc1599..90019968df 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, customerEntityPermissionChecker); + put(Resource.DEVICE, customerEntityPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,29 +85,6 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = - new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, - Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, - Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, - Operation.WRITE_CALCULATED_FIELD) { - - @Override - @SuppressWarnings("unchecked") - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - - if (!super.hasPermission(user, operation, entityId, entity)) { - return false; - } - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - if (!(entity instanceof HasCustomerId)) { - return false; - } - return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); - } - }; - private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index c730637148..de11521e85 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -30,8 +30,8 @@ public class TenantAdminPermissions extends AbstractPermissions { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, tenantEntityPermissionChecker); + put(Resource.DEVICE, tenantEntityPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -55,17 +55,6 @@ public class TenantAdminPermissions extends AbstractPermissions { public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { - @Override - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); - } - }; - - public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { - @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index baf20219ac..7d4d5300f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Asset Profile is prohibited!"); - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index 9bb1abc3c8..d99eaa2cea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -63,9 +63,6 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; - @Autowired - private CalculatedFieldService calculatedFieldService; - private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -484,21 +481,4 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } - @Test - public void testDeleteDeviceProfileIfCalculatedFieldExists() { - DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - CalculatedField calculatedField = new CalculatedField(); - calculatedField.setTenantId(tenantId); - calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); - calculatedField.setEntityId(savedDeviceProfile.getId()); - calculatedFieldService.save(calculatedField); - - assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Device Profile is prohibited!"); - } - } From 3c073bad6e65d7f07e7fee4b1cbc9ecbacf3b9ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:11:34 +0200 Subject: [PATCH 009/281] removed authority customer user for endpoints --- .../server/controller/CalculatedFieldController.java | 4 ++-- .../server/dao/device/DeviceProfileServiceImpl.java | 4 ++-- .../server/dao/service/AssetProfileServiceTest.java | 1 + .../server/dao/service/DeviceProfileServiceTest.java | 5 +---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index b3f2c018cb..78331e61b4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -57,7 +57,7 @@ public class CalculatedFieldController extends BaseController { "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @@ -70,7 +70,7 @@ public class CalculatedFieldController extends BaseController { @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." ) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) @ResponseBody public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index ec68562f5d..09e2be01e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService Date: Wed, 6 Nov 2024 15:21:28 +0200 Subject: [PATCH 010/281] added CF config and converter for it --- .../controller/CalculatedFieldController.java | 27 ++++ .../CalculatedFieldControllerTest.java | 38 ++++-- .../CalculatedFieldService.java | 2 + .../calculated_field/CalculatedField.java | 7 +- .../CalculatedFieldConfig.java | 43 +++++++ .../calculated_field/CalculatedFieldLink.java | 8 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 35 ++++-- .../CalculatedFieldConfigUtil.java | 117 ++++++++++++++++++ .../calculated_field/CalculatedFieldDao.java | 2 + .../CalculatedFieldLinkDao.java | 4 +- .../dao/customer/CustomerServiceImpl.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 6 +- .../model/sql/CalculatedFieldLinkEntity.java | 6 +- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 + .../JpaCalculatedFieldDao.java | 5 + .../JpaCalculatedFieldLinkDao.java | 6 +- .../server/dao/service/AssetServiceTest.java | 40 +++++- .../service/CalculatedFieldServiceTest.java | 52 +++++--- .../dao/service/CustomerServiceTest.java | 57 ++++++++- .../server/dao/service/DeviceServiceTest.java | 34 ++++- 23 files changed, 431 insertions(+), 74 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 78331e61b4..c33fb86ef8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,14 +27,24 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -46,6 +56,9 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { + private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( + EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); + private final CalculatedFieldService calculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -64,6 +77,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + checkReferencedEntities(calculatedField.getConfiguration()); return calculatedFieldService.save(calculatedField); } @@ -97,4 +111,17 @@ public class CalculatedFieldController extends BaseController { calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } + private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { + throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); + } + checkEntityId(referencedEntityId, Operation.READ); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index d9f4ac0377..52271bef83 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -18,15 +18,18 @@ package org.thingsboard.server.controller; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -75,7 +78,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); - assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); savedCalculatedField.setName("Test CF"); @@ -123,19 +126,28 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index f271f4ed24..b60c25e749 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -36,4 +36,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index 0bd8ded8f4..2482a7a19a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -47,8 +47,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; @Getter @Setter private Long version; @@ -64,8 +64,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { - super(); + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java new file mode 100644 index 0000000000..b1d63a9396 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Map; + +@Data +public class CalculatedFieldConfig { + + private Map arguments; + private Output output; + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private int defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java index 79f05c9b58..f24746abc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -37,8 +37,8 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; - @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; public CalculatedFieldLink() { super(); @@ -48,11 +48,11 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { this.tenantId = tenantId; this.entityId = entityId; - this.configuration = configuration; this.calculatedFieldId = calculatedFieldId; + this.configuration = configuration; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index da86d82cf0..f4c68aa15b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -222,8 +222,8 @@ public class BaseAssetService extends AbstractCachedEntityService !referencedEntityId.equals(calculatedField.getEntityId())) + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfig::getArguments) + .flatMap(arguments -> arguments.values().stream()) + .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -142,22 +153,26 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); - saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + CalculatedFieldLink existingLink = (calculatedField.getId() != null) + ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) + : null; + + CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); + saveCalculatedFieldLink(tenantId, updatedLink); } - private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); - return calculatedFieldLink; + private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { + CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(calculatedField.getEntityId()); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(calculatedField.getConfiguration()); + return link; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java new file mode 100644 index 0000000000..8f9422b354 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class CalculatedFieldConfigUtil { + + public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null) { + return null; + } + try { + CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); + Map arguments = new HashMap<>(); + + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + if (argumentNode.has("entityType") && argumentNode.has("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + + if (argumentNode.has("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); + } + + arguments.put(key, argument); + }); + } + calculatedFieldConfig.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + calculatedFieldConfig.setOutput(output); + } + + return calculatedFieldConfig; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); + } + } + + public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { + if (calculatedFieldConfig == null) { + return null; + } + try { + ObjectNode configNode = JacksonUtil.newObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + calculatedFieldConfig.getArguments().forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (calculatedFieldConfig.getOutput() != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", calculatedFieldConfig.getOutput().getType()); + outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); + } + + return configNode; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index d0d0a3c386..e4ec365f46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -26,6 +26,8 @@ public interface CalculatedFieldDao extends Dao { boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + List findAllByTenantId(TenantId tenantId); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 1c422311eb..380a3b4d96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -16,12 +16,14 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import java.util.UUID; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 2067ef4a5d..955d16c8e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -184,6 +184,10 @@ public class CustomerServiceImpl extends AbstractCachedEntityService implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration(); + this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -107,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(configuration); + calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 9f2efb230b..46c81e4449 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -34,6 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -75,7 +77,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity { - CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java index 17d7c0a9c6..d446d59859 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -25,6 +25,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantId(UUID tenantId); + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 9a0eef26c3..8a873e1580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -45,6 +45,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 0721e08d7f..4424f5bf42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -20,6 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -37,8 +39,8 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao { assetService.deleteAsset(tenantId, asset.getId()); }); + assets.forEach((asset) -> { + assetService.deleteAsset(tenantId, asset.getId()); + }); } } @@ -854,23 +858,49 @@ public class AssetServiceTest extends AbstractServiceTest { } @Test - public void testDeleteAssetIfCalculatedFieldExists() { + public void testDeleteAssetIfReferencedInCalculatedField() { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("My asset"); asset.setType("default"); Asset savedAsset = assetService.saveAsset(asset); + Asset assetWithCf = new Asset(); + assetWithCf.setTenantId(tenantId); + assetWithCf.setName("Asset with CF"); + assetWithCf.setType("default"); + Asset savedAssetWithCf = assetService.saveAsset(assetWithCf); + CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(savedAsset.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(savedAssetWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedAsset.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete asset that has entity views or calculated fields!"); + .hasMessage("Can't delete asset that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 46cc478904..32b7cbc00a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -21,17 +21,19 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import java.util.Map; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -60,7 +62,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -84,7 +86,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatesFieldWithNonExistingDeviceId() { - CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); + CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) .isInstanceOf(IllegalArgumentException.class) @@ -94,7 +97,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) @@ -105,7 +108,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingExternalId() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); calculatedFieldService.save(calculatedField); @@ -147,28 +150,18 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); return calculatedFieldService.save(calculatedField); } - private CalculatedField getCalculatedField(DeviceId deviceId) { + private CalculatedField getCalculatedField(EntityId entityId, EntityId referencedEntityId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(deviceId); + calculatedField.setEntityId(entityId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -177,11 +170,30 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); +// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + private Device createTestDevice() { Device device = new Device(); device.setTenantId(tenantId); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0f4c139ce8..c021512a44 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -30,14 +30,20 @@ import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -45,13 +51,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CustomerServiceTest extends AbstractServiceTest { @Autowired CustomerService customerService; + @Autowired + CalculatedFieldService calculatedFieldService; + @Autowired + AssetService assetService; static final int TIMEOUT = 30; @@ -343,4 +353,49 @@ public class CustomerServiceTest extends AbstractServiceTest { } } + @Test + public void testDeleteCustomerIfReferencedInCalculatedField() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("My customer"); + Customer savedCustomer = customerService.saveCustomer(customer); + + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedCustomer.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> customerService.deleteCustomer(tenantId, savedCustomer.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete customer that is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2508090b2..a9ed4d201f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -66,6 +67,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -1203,19 +1205,41 @@ public class DeviceServiceTest extends AbstractServiceTest { } @Test - public void testDeleteDeviceIfCalculatedFieldExists() { - Device device = saveDevice(tenantId, "Test"); + public void testDeleteAssetIfReferencedInCalculatedField() { + Device device = saveDevice(tenantId, "Test Device"); + Device deviceWithCf = saveDevice(tenantId, "Device with CF"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(device.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(deviceWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(device.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete device that has entity views or calculated fields!"); + .hasMessage("Can't delete device that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } + } From 47e7c0f3a535aaa16d2d4b9bd74fe9f1f1f556f4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 15:25:07 +0200 Subject: [PATCH 011/281] removed permission for customer --- .../service/security/permission/CustomerUserPermissions.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 90019968df..46cd1c2979 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -188,8 +188,7 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( - Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { @Override @SuppressWarnings("unchecked") @@ -203,5 +202,4 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; - } From 258ba4807ba0c514091a8bc2f8a475cf6699b701 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 16:29:20 +0200 Subject: [PATCH 012/281] fixed entity service registry test --- .../server/dao/entity/DefaultEntityServiceRegistry.java | 3 +++ .../server/dao/service/EntityServiceRegistryTest.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index 364d3a12f3..56e8b95473 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -43,6 +43,9 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { if (EntityType.RULE_CHAIN.equals(entityType)) { entityDaoServicesMap.put(EntityType.RULE_NODE, entityDaoService); } + if (EntityType.CALCULATED_FIELD.equals(entityType)) { + entityDaoServicesMap.put(EntityType.CALCULATED_FIELD_LINK, entityDaoService); + } }); log.debug("Initialized EntityServiceRegistry total [{}] entries", entityDaoServicesMap.size()); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index c3f6213398..0769479f48 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; @@ -44,4 +45,8 @@ public class EntityServiceRegistryTest extends AbstractServiceTest { Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.RULE_NODE) instanceof RuleChainService); } + @Test + public void givenCalculatedFieldLinkEntityType_whenGetServiceByEntityTypeCalled_thenReturnedCalculatedFieldService() { + Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.CALCULATED_FIELD_LINK) instanceof CalculatedFieldService); + } } From 1d702365381db46e7a925043973ba2d5fe021134 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 09:14:41 +0200 Subject: [PATCH 013/281] fixed entity dao registry test --- .../dao/calculated_field/CalculatedFieldLinkDao.java | 2 -- .../dao/sql/calculated_field/JpaCalculatedFieldDao.java | 7 +++++++ .../sql/calculated_field/JpaCalculatedFieldLinkDao.java | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 380a3b4d96..7fa0285b9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -20,8 +20,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; -import java.util.UUID; - public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 8a873e1580..affd652514 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -20,6 +20,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -65,4 +66,10 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getRepository() { return calculatedFieldRepository; } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 4424f5bf42..f77baa4e42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -53,4 +54,9 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao Date: Thu, 7 Nov 2024 12:02:39 +0200 Subject: [PATCH 014/281] renamed directories --- .../thingsboard/server/controller/BaseController.java | 4 ++-- .../server/controller/CalculatedFieldController.java | 8 +++----- .../processor/CalculatedFieldsDeletionTaskProcessor.java | 2 +- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldService.java | 6 +++--- .../data/{calculated_field => cf}/CalculatedField.java | 3 +-- .../{calculated_field => cf}/CalculatedFieldConfig.java | 2 +- .../{calculated_field => cf}/CalculatedFieldLink.java | 3 +-- .../BaseCalculatedFieldService.java | 9 ++++----- .../CalculatedFieldConfigUtil.java | 4 ++-- .../dao/{calculated_field => cf}/CalculatedFieldDao.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldLinkDao.java | 4 ++-- .../server/dao/entity/AbstractEntityService.java | 2 +- .../server/dao/model/sql/CalculatedFieldEntity.java | 6 +++--- .../server/dao/model/sql/CalculatedFieldLinkEntity.java | 6 +++--- .../service/validator/CalculatedFieldDataValidator.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidator.java | 4 ++-- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 +- .../{calculated_field => cf}/JpaCalculatedFieldDao.java | 6 +++--- .../JpaCalculatedFieldLinkDao.java | 6 +++--- .../thingsboard/server/dao/service/AssetServiceTest.java | 6 +++--- .../server/dao/service/CalculatedFieldServiceTest.java | 8 ++++---- .../server/dao/service/CustomerServiceTest.java | 6 +++--- .../server/dao/service/DeviceServiceTest.java | 6 +++--- .../server/dao/service/EntityServiceRegistryTest.java | 2 +- .../validator/CalculatedFieldDataValidatorTest.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidatorTest.java | 4 ++-- 28 files changed, 61 insertions(+), 66 deletions(-) rename common/dao-api/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldService.java (87%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedField.java (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldConfig.java (94%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldLink.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/BaseCalculatedFieldService.java (96%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldConfigUtil.java (97%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldDao.java (89%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldLinkDao.java (88%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldLinkRepository.java (94%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldRepository.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldDao.java (92%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldLinkDao.java (91%) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 6d52bdb358..eb6fec88b5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,7 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -126,7 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index c33fb86ef8..2e9989334d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -28,23 +28,21 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java index 18d84b02ce..2362081ff9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; @Component @RequiredArgsConstructor diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 52271bef83..affab6ee8b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,8 +21,8 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java similarity index 87% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b60c25e749..8171759def 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java similarity index 97% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 2482a7a19a..5c52ad0609 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java index b1d63a9396..b51258b2ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java similarity index 95% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index f24746abc0..2f176b13d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0b2ba43fbb..fe4422ee71 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -37,7 +37,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; -import java.util.Objects; import java.util.Optional; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java similarity index 97% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 8f9422b354..34dd885e12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java similarity index 89% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index e4ec365f46..b755d4b6c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java similarity index 88% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 7fa0285b9a..95259e75c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index cda9222f76..7cf7b445d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 43b4f2fc9f..98eb050d28 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +33,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 46c81e4449..73a2f9fbdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index e50f2c902a..80e421f350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java index 83c30d4d03..2b11f8fe42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index a4f035c2c9..fa1c5ce38e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index d446d59859..333057e8c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java similarity index 92% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index affd652514..3fbfe92efe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; @@ -21,11 +21,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f77baa4e42..b9ed6dc3a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 39386e4668..ba0c73fb09 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -41,7 +41,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 32b7cbc00a..632e4b19e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,13 +23,13 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index c021512a44..548a5bfcf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,13 +31,13 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index a9ed4d201f..16bb08350b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,8 +39,8 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index 0769479f48..147e69d1cb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,7 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index bc7990ca64..6d0eb0ad38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -19,10 +19,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index c9fa57a751..5ec6841150 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -19,11 +19,11 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; From 4dbc35273ae9182024829ca558336b92fce5de6d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 13:26:48 +0200 Subject: [PATCH 015/281] added calculated field service to ctx --- .../server/actors/ActorSystemContext.java | 5 +++++ .../actors/ruleChain/DefaultTbContext.java | 6 ++++++ .../server/dao/cf/CalculatedFieldService.java | 3 +++ .../dao/cf/BaseCalculatedFieldService.java | 9 +++++++++ .../thingsboard/rule/engine/api/TbContext.java | 3 +++ .../rule/engine/util/TenantIdLoader.java | 14 ++++++++++++++ .../rule/engine/util/TenantIdLoaderTest.java | 17 +++++++++++++++++ 7 files changed, 57 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 5966716ed8..400374c731 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -62,6 +62,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -389,6 +390,10 @@ public class ActorSystemContext { @Getter private SlackService slackService; + @Autowired + @Getter + private CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ec10402821..274d50a6c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -78,6 +78,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -848,6 +849,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getSlackService(); } + @Override + public CalculatedFieldService getCalculatedFieldService() { + return mainCtx.getCalculatedFieldService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 8171759def..b2da1d3fe4 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -34,6 +35,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fe4422ee71..be21c9b887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -114,6 +115,14 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkById, tenantId [{}], calculatedFieldLinkId [{}]", tenantId, calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 7dd4505f29..22c7ced237 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -357,6 +358,8 @@ public interface TbContext { SlackService getSlackService(); + CalculatedFieldService getCalculatedFieldService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 1e1b031259..98724b7b68 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -18,10 +18,13 @@ package org.thingsboard.rule.engine.util; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -157,6 +160,17 @@ public class TenantIdLoader { case MOBILE_APP: tenantEntity = ctx.getMobileAppService().findMobileAppById(ctxTenantId, new MobileAppId(id)); break; + case CALCULATED_FIELD: + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, new CalculatedFieldId(id)); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = ctx.getCalculatedFieldService().findCalculatedFieldLinkById(ctxTenantId, new CalculatedFieldLinkId(id)); + if (calculatedFieldLink != null) { + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, calculatedFieldLink.getCalculatedFieldId()); + } else { + tenantEntity = null; + } + break; default: throw new RuntimeException("Unexpected entity type: " + entityId.getEntityType()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 3e0e479c80..2234e50a16 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -43,6 +43,8 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -66,6 +68,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; @@ -151,6 +154,8 @@ public class TenantIdLoaderTest { private DomainService domainService; @Mock private MobileAppService mobileAppService; + @Mock + private CalculatedFieldService calculatedFieldService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -392,6 +397,18 @@ public class TenantIdLoaderTest { when(ctx.getMobileAppService()).thenReturn(mobileAppService); doReturn(mobileApp).when(mobileAppService).findMobileAppById(eq(tenantId), any()); break; + case CALCULATED_FIELD: + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedField).when(calculatedFieldService).findById(eq(tenantId), any()); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } From bcef71f78d6ceb6a36703aa11660ea9a0383df06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 14:07:58 +0200 Subject: [PATCH 016/281] added tbCalculatedField Service --- .../controller/CalculatedFieldController.java | 39 +---- .../entitiy/AbstractTbEntityService.java | 47 ++++++ .../cf/DefaultTbCalculatedFieldService.java | 149 ++++++++++++++++++ .../entitiy/cf/TbCalculatedFieldService.java | 33 ++++ .../server/dao/cf/CalculatedFieldService.java | 6 + .../dao/cf/BaseCalculatedFieldService.java | 42 ++--- .../server/dao/cf/CalculatedFieldDao.java | 2 + .../server/dao/cf/CalculatedFieldLinkDao.java | 4 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 + .../service/CalculatedFieldServiceTest.java | 10 -- 11 files changed, 270 insertions(+), 73 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 2e9989334d..4bf243034b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,23 +27,14 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; - import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -54,10 +45,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { - private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( - EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); - - private final CalculatedFieldService calculatedFieldService; + private final TbCalculatedFieldService tbCalculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -75,8 +63,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - checkReferencedEntities(calculatedField.getConfiguration()); - return calculatedFieldService.save(calculatedField); + return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); } @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", @@ -88,7 +75,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkNotNull(calculatedField); checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); return calculatedField; @@ -103,23 +90,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - TenantId tenantId = getTenantId(); - CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); - } - - private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); - for (EntityId referencedEntityId : referencedEntityIds) { - if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { - throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); - } - checkEntityId(referencedEntityId, Operation.READ); - } + tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 438cb58b91..2b19d34e57 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -25,16 +25,30 @@ import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.util.ThrowingBiFunction; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -43,6 +57,8 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import static org.thingsboard.server.dao.service.Validator.validateId; + @Slf4j public abstract class AbstractTbEntityService { @@ -71,6 +87,18 @@ public abstract class AbstractTbEntityService { @Autowired(required = false) @Lazy private EntitiesVersionControlService vcService; + @Autowired + protected AccessControlService accessControlService; + @Autowired + protected TenantService tenantService; + @Autowired + protected AssetService assetService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected AssetProfileService assetProfileService; + @Autowired + protected DeviceProfileService deviceProfileService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); @@ -120,4 +148,23 @@ public abstract class AbstractTbEntityService { return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); } } + + protected & HasTenantId, I extends EntityId> E checkEntityId(I entityId, ThrowingBiFunction findingFunction, Operation operation, SecurityUser user) throws Exception { + try { + validateId((UUIDBased) entityId, "Invalid entity id"); + E entity = findingFunction.apply(user.getTenantId(), entityId); + checkNotNull(entity, entityId.getEntityType().getNormalName() + " with id [" + entityId + "] is not found"); + return checkEntity(user, entity, operation); + } catch (Exception e) { + throw e; + } + } + + + protected & HasTenantId, I extends EntityId> E checkEntity(SecurityUser user, E entity, Operation operation) throws ThingsboardException { + checkNotNull(entity, "Entity not found"); + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), entity); + return entity; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java new file mode 100644 index 0000000000..a7c77cf4bc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.service.Validator.validateEntityId; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + + private final Map calculatedFields; + private final Map calculatedFieldLinks; + private final CalculatedFieldService calculatedFieldService; + + public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { + this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); + this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); + this.calculatedFieldService = calculatedFieldService; + } + + @Override + public void onMsg() { + + } + + @Override + public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { + ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + TenantId tenantId = calculatedField.getTenantId(); + try { + checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkReferencedEntities(calculatedField.getConfiguration(), user); + CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); + logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); + return savedCalculatedField; + } catch (ThingsboardException e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); + throw e; + } + } + + @Override + public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { + return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + } + + @Override + @Transactional + public void delete(CalculatedField calculatedField, SecurityUser user) { + ActionType actionType = ActionType.DELETED; + TenantId tenantId = calculatedField.getTenantId(); + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + try { + calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); + logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); + throw e; + } + } + + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } + } + + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); + E entity = findEntity(user.getTenantId(), referencedEntityId); + checkNotNull(entity); + checkEntity(user, entity, Operation.READ); + } + + } + + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { + return (E) switch (entityId.getEntityType()) { + case TENANT -> tenantService.findTenantById((TenantId) entityId); + case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); + case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); + case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java new file mode 100644 index 0000000000..2dc1cc35b2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface TbCalculatedFieldService { + + void onMsg(); + + CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; + + CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + + void delete(CalculatedField calculatedField, SecurityUser user); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b2da1d3fe4..7760c49d2d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -23,12 +23,16 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.List; + public interface CalculatedFieldService extends EntityDaoService { CalculatedField save(CalculatedField calculatedField); CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); @@ -37,6 +41,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index be21c9b887..ecc5aecce1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -22,19 +22,11 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -53,10 +45,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; private final CalculatedFieldLinkDao calculatedFieldLinkDao; - private final DeviceService deviceService; - private final AssetService assetService; - private final DeviceProfileService deviceProfileService; - private final AssetProfileService assetProfileService; private final DataValidator calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -65,7 +53,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); - checkEntityExistence(tenantId, calculatedField.getEntityId()); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); @@ -86,6 +73,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public List findAll() { + log.trace("Executing findAll"); + return calculatedFieldDao.findAll(); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -123,6 +116,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public List findAllCalculatedFieldLinks() { + log.trace("Executing findAllCalculatedFieldLinks"); + return calculatedFieldLinkDao.findAll(); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -148,23 +147,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { - switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); - default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); - } - } - private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldLink existingLink = (calculatedField.getId() != null) ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index b755d4b6c0..bb52d0e2c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -28,6 +28,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findAll(); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 95259e75c4..c4a06c88a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -20,8 +20,12 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 3fbfe92efe..4bf4c1ead7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -51,6 +51,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index b9ed6dc3a4..f584a8d76c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -44,6 +45,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 632e4b19e6..c4ad2856a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -84,16 +84,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - @Test - public void testSaveCalculatesFieldWithNonExistingDeviceId() { - DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); - CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); - - assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); - } - @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); From 86dd8e725bf701f0c0fe561c3e2b6e8729235305 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 11 Nov 2024 17:29:24 +0200 Subject: [PATCH 017/281] added onCalculatedFieldAdded implementation --- .../entitiy/AbstractTbEntityService.java | 3 + .../entitiy/cf/CalculatedFieldCtx.java | 31 ++ .../entitiy/cf/CalculatedFieldState.java | 30 ++ .../cf/DefaultTbCalculatedFieldService.java | 281 ++++++++++++++++-- .../entitiy/cf/TbCalculatedFieldService.java | 8 +- .../CalculatedFieldControllerTest.java | 2 +- .../server/dao/asset/AssetService.java | 2 + .../server/dao/cf/CalculatedFieldService.java | 10 +- .../server/dao/device/DeviceService.java | 2 + .../server/dao/entity/EntityService.java | 3 + .../cf/BaseCalculatedFieldConfiguration.java | 158 ++++++++++ .../common/data/cf/CalculatedField.java | 8 +- .../data/cf/CalculatedFieldConfiguration.java | 48 +++ .../common/data/cf/CalculatedFieldLink.java | 4 +- .../cf/CalculatedFiledLinkConfiguration.java | 29 ++ .../SimpleCalculatedFieldConfiguration.java | 39 +++ common/proto/src/main/proto/queue.proto | 21 ++ .../server/dao/asset/AssetDao.java | 10 + .../server/dao/asset/BaseAssetService.java | 9 + .../dao/cf/BaseCalculatedFieldService.java | 60 ++-- .../dao/cf/CalculatedFieldConfigUtil.java | 172 +++++------ .../server/dao/cf/CalculatedFieldDao.java | 4 + .../server/dao/cf/CalculatedFieldLinkDao.java | 6 +- .../server/dao/device/DeviceDao.java | 12 + .../server/dao/device/DeviceServiceImpl.java | 9 + .../server/dao/entity/BaseEntityService.java | 5 + .../dao/model/sql/CalculatedFieldEntity.java | 18 +- .../model/sql/CalculatedFieldLinkEntity.java | 9 +- .../server/dao/sql/asset/AssetRepository.java | 10 + .../server/dao/sql/asset/JpaAssetDao.java | 11 + .../sql/cf/CalculatedFieldLinkRepository.java | 3 +- ...efaultNativeCalculatedFieldRepository.java | 146 +++++++++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 11 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 14 +- .../cf/NativeCalculatedFieldRepository.java | 29 ++ .../dao/sql/device/DeviceRepository.java | 8 + .../server/dao/sql/device/JpaDeviceDao.java | 11 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 41 files changed, 1085 insertions(+), 159 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 2b19d34e57..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -99,6 +100,8 @@ public abstract class AbstractTbEntityService { protected AssetProfileService assetProfileService; @Autowired protected DeviceProfileService deviceProfileService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java new file mode 100644 index 0000000000..61f28304f2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@Builder +public class CalculatedFieldCtx { + + private final CalculatedFieldId calculatedFieldId; + private final EntityId entityId; + private final CalculatedFieldState state; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java new file mode 100644 index 0000000000..dc07b820ea --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class CalculatedFieldState { + + Map arguments; + String result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index a7c77cf4bc..836f17650e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,36 +15,64 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -52,21 +80,213 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @TbCoreComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private final Map calculatedFields; - private final Map calculatedFieldLinks; private final CalculatedFieldService calculatedFieldService; + private final TbDeviceProfileCache deviceProfileCache; + private final TbAssetProfileCache assetProfileCache; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private ListeningScheduledExecutorService scheduledExecutor; - public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { - this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); - this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); - this.calculatedFieldService = calculatedFieldService; + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${state.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Value("10") + @Getter + private int defaultCalculatedFieldCheckIntervalInSec; + + @PostConstruct + public void init() { + // from AbstractPartitionBasedService + scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); + /// + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + // from AbstractPartitionBasedService + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + } + /// + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { + return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); + } + + private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { + return timeseriesService.findLatest(tenantId, entityId, keys); + } + + private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { + ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); + ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); + + ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); + + return Futures.transform(combinedFuture, results -> { + List attributes = (List) results.get(0); + List timeSeries = (List) results.get(1); + + initializeState(calculatedField, attributes, timeSeries); + + return null; + }, MoreExecutors.directExecutor()); + } + + private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + String calculation = performCalculation(state.getArguments()); + + Map updatedArguments = state.getArguments(); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + Map arguments = calculatedField.getConfiguration().getArguments(); + + Map argumentValues = arguments.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) + )); + + String calculation = performCalculation(argumentValues); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + + calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, + List attributes, List timeSeries) { + String type = argument.getType(); + String value = null; + + if ("ATTRIBUTES".equals(type)) { + value = attributes.stream() + .filter(attribute -> attribute.getKey().equals(key)) + .map(AttributeKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } else if ("TIME_SERIES".equals(type)) { + value = timeSeries.stream() + .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) + .map(TsKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } + + return value != null ? value : argument.getDefaultValue(); + } + + @Override + public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE: { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + case ASSET_PROFILE: { + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + for (AssetId assetId : assetIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + case DEVICE_PROFILE: { + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + for (DeviceId deviceId : deviceIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field or entity was probably deleted while message was in queue; + callback.onSuccess(); + } + } catch (Exception e) { + log.trace("Failed to process calculated field add msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override - public void onMsg() { + public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + } catch (Exception e) { + log.trace("Failed to process calculated field update msg: [{}]", proto, e); + callback.onFailure(e); + } + } + @Override + public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { + try { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.remove(calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field delete msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override @@ -105,28 +325,26 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + // TODO: read all states(CalculatedFieldCtx) + states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } - private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); for (EntityId referencedEntityId : referencedEntityIds) { validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); E entity = findEntity(user.getTenantId(), referencedEntityId); @@ -137,13 +355,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { - return (E) switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById((TenantId) entityId); - case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); - case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); - case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + return switch (entityId.getEntityType()) { + case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } + private String performCalculation(Map argumentValues) { + return "calculation"; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 2dc1cc35b2..4f14270c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,11 +18,17 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onMsg(); + void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); + + void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); + + void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index affab6ee8b..3f505c78ef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -126,7 +126,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(null)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 7252db8097..2a9fe08827 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -63,6 +63,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 7760c49d2d..c12acade6c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; @@ -31,7 +33,9 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); + List findAllCalculatedFields(); + + PageData findAllCalculatedFields(PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -43,6 +47,10 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinks(); + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 005c740571..0848e5a1cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -75,6 +75,8 @@ public interface DeviceService extends EntityDaoService { PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 5f522121d7..fc8859eceb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..9bb6d59428 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @JsonIgnore + private final ObjectMapper mapper = new ObjectMapper(); + + protected Map arguments; + protected SimpleCalculatedFieldConfiguration.Output output; + + public BaseCalculatedFieldConfiguration() { + } + + public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); + this.arguments = calculatedFieldConfig.getArguments(); + this.output = calculatedFieldConfig.getOutput(); + } + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + arguments.values().stream() + .filter(argument -> argument.getEntityId().equals(entityId)) + .forEach(argument -> { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().add(argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().add(argument.getKey()); + break; + } + }); + + return linkConfiguration; + } + + @Override + public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { + ObjectNode configNode = mapper.createObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + arguments.forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (output != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", output.getType()); + outputNode.put("expression", output.getExpression()); + } + + return configNode; + } + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + + private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null || !config.isObject()) { + return null; + } + + Map arguments = new HashMap<>(); + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + Argument argument = new Argument(); + if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + arguments.put(key, argument); + }); + } + this.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + Output output = new Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + this.setOutput(output); + } + + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 5c52ad0609..d96de37a39 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serializable; + @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -46,8 +48,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema - private transient CalculatedFieldConfig configuration; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; @Getter @Setter private Long version; @@ -63,7 +65,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e599b8c4d3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") +}) +public interface CalculatedFieldConfiguration { + + String getType(); + + Map getArguments(); + + List getReferencedEntities(); + + CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + + JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 2f176b13d2..922fda1f34 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFieldConfig configuration; + private transient CalculatedFiledLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java new file mode 100644 index 0000000000..26d867fd7f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class CalculatedFiledLinkConfiguration { + + private List attributes = new ArrayList<>(); + private List timeSeries = new ArrayList<>(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..d635e0d82e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public SimpleCalculatedFieldConfiguration() { + super(); + } + + public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public String getType() { + return "SIMPLE"; + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 875a531566..e1d6320779 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,6 +1267,27 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } +message CalculatedFieldAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldDeleteMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + /** TB Core to Version Control Service */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..8e40ec6d87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -103,6 +103,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f4c68aa15b..2135792174 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -285,6 +285,15 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index ecc5aecce1..0210f6d70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -20,20 +20,24 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @@ -74,11 +78,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public List findAll() { + public List findAllCalculatedFields() { log.trace("Executing findAll"); return calculatedFieldDao.findAll(); } + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -122,6 +133,19 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findAll(); } + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -132,9 +156,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findAllByTenantId(tenantId).stream() .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfig::getArguments) - .flatMap(arguments -> arguments.values().stream()) - .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } @Override @@ -148,21 +171,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink existingLink = (calculatedField.getId() != null) - ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) - : null; - - CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); - saveCalculatedFieldLink(tenantId, updatedLink); + List links = buildCalculatedFieldLinks(tenantId, calculatedField); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { - CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(calculatedField.getEntityId()); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(calculatedField.getConfiguration()); - return link; + private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); + return cfConfig.getReferencedEntities().stream() + .map(referencedEntityId -> { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); + return link; + }) + .collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 34dd885e12..980fff6273 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -29,89 +31,89 @@ import java.util.UUID; public class CalculatedFieldConfigUtil { - public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null) { - return null; - } - try { - CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); - Map arguments = new HashMap<>(); - - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); - if (argumentNode.has("entityType") && argumentNode.has("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } - argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); - - if (argumentNode.has("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); - } - - arguments.put(key, argument); - }); - } - calculatedFieldConfig.setArguments(arguments); - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); - output.setType(outputNode.get("type").asText()); - output.setExpression(outputNode.get("expression").asText()); - calculatedFieldConfig.setOutput(output); - } - - return calculatedFieldConfig; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); - } - } - - public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { - if (calculatedFieldConfig == null) { - return null; - } - try { - ObjectNode configNode = JacksonUtil.newObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - calculatedFieldConfig.getArguments().forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); - argumentNode.put("defaultValue", argument.getDefaultValue()); - }); - - if (calculatedFieldConfig.getOutput() != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("type", calculatedFieldConfig.getOutput().getType()); - outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); - } - - return configNode; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); - } - } +// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { +// if (config == null) { +// return null; +// } +// try { +// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); +// Map arguments = new HashMap<>(); +// +// JsonNode argumentsNode = config.get("arguments"); +// if (argumentsNode != null && argumentsNode.isObject()) { +// argumentsNode.fields().forEachRemaining(entry -> { +// String key = entry.getKey(); +// JsonNode argumentNode = entry.getValue(); +// +// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); +// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { +// String referencedEntityType = argumentNode.get("entityType").asText(); +// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); +// } else { +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); +// } +// argument.setKey(argumentNode.get("key").asText()); +// argument.setType(argumentNode.get("type").asText()); +// +// if (argumentNode.has("defaultValue")) { +// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); +// } +// +// arguments.put(key, argument); +// }); +// } +// calculatedFieldConfig.setArguments(arguments); +// +// JsonNode outputNode = config.get("output"); +// if (outputNode != null) { +// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); +// output.setType(outputNode.get("type").asText()); +// output.setExpression(outputNode.get("expression").asText()); +// calculatedFieldConfig.setOutput(output); +// } +// +// return calculatedFieldConfig; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); +// } +// } +// +// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { +// if (calculatedFieldConfig == null) { +// return null; +// } +// try { +// ObjectNode configNode = JacksonUtil.newObjectNode(); +// +// ObjectNode argumentsNode = configNode.putObject("arguments"); +// calculatedFieldConfig.getArguments().forEach((key, argument) -> { +// ObjectNode argumentNode = argumentsNode.putObject(key); +// EntityId referencedEntityId = argument.getEntityId(); +// if (referencedEntityId != null) { +// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); +// argumentNode.put("entityId", referencedEntityId.getId().toString()); +// } else { +// argumentNode.put("entityType", entityType.name()); +// argumentNode.put("entityId", entityId.toString()); +// } +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", argument.getType()); +// argumentNode.put("defaultValue", argument.getDefaultValue()); +// }); +// +// if (calculatedFieldConfig.getOutput() != null) { +// ObjectNode outputNode = configNode.putObject("output"); +// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); +// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); +// } +// +// return configNode; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); +// } +// } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index bb52d0e2c2..4abe02a09b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; @@ -30,6 +32,8 @@ public interface CalculatedFieldDao extends Dao { List findAll(); + PageData findAll(PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index c4a06c88a3..728e19b890 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,14 +18,18 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); + PageData findAll(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..7c82df7703 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +87,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ac6b8330a9..ffebd5f032 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -395,6 +395,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..5efea9a6df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -134,6 +134,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 98eb050d28..9cb4f08640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,6 +24,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +35,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -93,7 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -117,4 +117,14 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 73a2f9fbdf..3dc08d6bf1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,8 +22,10 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -65,7 +65,6 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..db6472f825 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -159,6 +160,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index fa1c5ce38e..61c4026cca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -18,10 +18,11 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import java.util.List; import java.util.UUID; public interface CalculatedFieldLinkRepository extends JpaRepository { - CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..8401d99128 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + String type = (String) row.get("type"); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + long version = (long) row.get("version"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setVersion(version); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFiledLinkConfiguration.class)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 4bf4c1ead7..1137b91947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,16 +18,20 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -40,6 +44,7 @@ import java.util.UUID; public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { @@ -56,6 +61,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f584a8d76c..a2f8f224c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -22,7 +22,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -39,10 +42,11 @@ import java.util.UUID; public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override - public CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); } @Override @@ -50,6 +54,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..76fed8c311 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..5093da78c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -81,6 +81,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..bf0def9fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -173,6 +173,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index ba0c73fb09..311bf23263 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -892,7 +892,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c4ad2856a9..8e50e1e0fb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -151,7 +151,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 548a5bfcf5..3f990b9dd7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -387,7 +387,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 16bb08350b..90d04d5216 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1230,7 +1230,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); From 444c5bf07976df4e8ac7d57fb870a120dd673551 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:25:59 +0200 Subject: [PATCH 018/281] refactored service and test classes --- .../cf/DefaultTbCalculatedFieldService.java | 48 ++++--- .../cf/BaseCalculatedFieldConfiguration.java | 12 +- .../common/data/cf/CalculatedFieldConfig.java | 43 ------- .../data/cf/CalculatedFieldConfiguration.java | 7 +- .../common/data/cf/CalculatedFieldLink.java | 4 +- ... => CalculatedFieldLinkConfiguration.java} | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 9 +- .../dao/cf/CalculatedFieldConfigUtil.java | 119 ------------------ .../dao/model/sql/CalculatedFieldEntity.java | 3 +- .../model/sql/CalculatedFieldLinkEntity.java | 4 +- ...efaultNativeCalculatedFieldRepository.java | 9 +- .../main/resources/sql/schema-entities.sql | 4 +- .../server/dao/service/AssetServiceTest.java | 12 +- .../service/CalculatedFieldServiceTest.java | 36 +++--- .../dao/service/CustomerServiceTest.java | 12 +- .../server/dao/service/DeviceServiceTest.java | 13 +- 16 files changed, 84 insertions(+), 253 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{CalculatedFiledLinkConfiguration.java => CalculatedFieldLinkConfiguration.java} (94%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 836f17650e..e222259a53 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -58,8 +58,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -84,8 +82,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final TbDeviceProfileCache deviceProfileCache; - private final TbAssetProfileCache assetProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private ListeningScheduledExecutorService scheduledExecutor; @@ -215,6 +211,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return value != null ? value : argument.getDefaultValue(); } + private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { + for (T profileId : profileIds) { + for (CalculatedFieldLink link : links) { + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -227,33 +232,24 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE: { + case ASSET, DEVICE -> { for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); } } - case ASSET_PROFILE: { + case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - for (AssetId assetId : assetIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, assetIds); } - case DEVICE_PROFILE: { + case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - for (DeviceId deviceId : deviceIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, deviceIds); } - default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } } else { //Calculated field or entity was probably deleted while message was in queue; @@ -336,8 +332,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) - .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } @@ -357,7 +354,8 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); - default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + default -> + throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 9bb6d59428..3894835abc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -37,7 +38,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; - protected SimpleCalculatedFieldConfiguration.Output output; + protected Output output; public BaseCalculatedFieldConfiguration() { } @@ -45,19 +46,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); - this.output = calculatedFieldConfig.getOutput(); + this.output = calculatedFieldConfig.getOutput(); } @Override public List getReferencedEntities() { return arguments.values().stream() - .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .map(Argument::getEntityId) + .filter(Objects::nonNull) .collect(Collectors.toList()); } @Override - public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { - CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.values().stream() .filter(argument -> argument.getEntityId().equals(entityId)) .forEach(argument -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java deleted file mode 100644 index b51258b2ec..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.Map; - -@Data -public class CalculatedFieldConfig { - - private Map arguments; - private Output output; - - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private int defaultValue; - } - - @Data - public static class Output { - private String type; - private String expression; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index e599b8c4d3..deaeebcb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; @@ -35,14 +36,18 @@ import java.util.UUID; }) public interface CalculatedFieldConfiguration { + @JsonIgnore String getType(); Map getArguments(); + @JsonIgnore List getReferencedEntities(); - CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore + CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 922fda1f34..f9f9ee625f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFiledLinkConfiguration configuration; + private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 26d867fd7f..02d668a67a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class CalculatedFiledLinkConfiguration { +public class CalculatedFieldLinkConfiguration { private List attributes = new ArrayList<>(); private List timeSeries = new ArrayList<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0210f6d70b..db5539eb96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -110,13 +110,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); - try { - log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); - return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); - } catch (Exception e) { - checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); - throw e; - } + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java deleted file mode 100644 index 980fff6273..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cf; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class CalculatedFieldConfigUtil { - -// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { -// if (config == null) { -// return null; -// } -// try { -// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); -// Map arguments = new HashMap<>(); -// -// JsonNode argumentsNode = config.get("arguments"); -// if (argumentsNode != null && argumentsNode.isObject()) { -// argumentsNode.fields().forEachRemaining(entry -> { -// String key = entry.getKey(); -// JsonNode argumentNode = entry.getValue(); -// -// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); -// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { -// String referencedEntityType = argumentNode.get("entityType").asText(); -// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); -// } else { -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); -// } -// argument.setKey(argumentNode.get("key").asText()); -// argument.setType(argumentNode.get("type").asText()); -// -// if (argumentNode.has("defaultValue")) { -// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); -// } -// -// arguments.put(key, argument); -// }); -// } -// calculatedFieldConfig.setArguments(arguments); -// -// JsonNode outputNode = config.get("output"); -// if (outputNode != null) { -// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); -// output.setType(outputNode.get("type").asText()); -// output.setExpression(outputNode.get("expression").asText()); -// calculatedFieldConfig.setOutput(output); -// } -// -// return calculatedFieldConfig; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); -// } -// } -// -// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { -// if (calculatedFieldConfig == null) { -// return null; -// } -// try { -// ObjectNode configNode = JacksonUtil.newObjectNode(); -// -// ObjectNode argumentsNode = configNode.putObject("arguments"); -// calculatedFieldConfig.getArguments().forEach((key, argument) -> { -// ObjectNode argumentNode = argumentsNode.putObject(key); -// EntityId referencedEntityId = argument.getEntityId(); -// if (referencedEntityId != null) { -// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); -// argumentNode.put("entityId", referencedEntityId.getId().toString()); -// } else { -// argumentNode.put("entityType", entityType.name()); -// argumentNode.put("entityId", entityId.toString()); -// } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", argument.getType()); -// argumentNode.put("defaultValue", argument.getDefaultValue()); -// }); -// -// if (calculatedFieldConfig.getOutput() != null) { -// ObjectNode outputNode = configNode.putObject("output"); -// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); -// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); -// } -// -// return configNode; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); -// } -// } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 9cb4f08640..725ddd7bef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -117,8 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - String type = config.get("type").asText(); + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { switch (type) { case "SIMPLE": return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 3dc08d6bf1..0ac5a9bc94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -86,7 +86,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field for such entity id is already exists!"); - } - private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); @@ -148,10 +140,10 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -160,22 +152,28 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); -// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { + CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); + calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); + return calculatedFieldLinkConfiguration; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3f990b9dd7..0e2d9b797b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -369,25 +369,25 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(savedAsset.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 90d04d5216..33b80c0d99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1212,25 +1212,25 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(deviceWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); @@ -1241,5 +1241,4 @@ public class DeviceServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - } From b6b7af8003d7737c8cc5f818c85711b338329db9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:51:29 +0200 Subject: [PATCH 019/281] refactored tests --- .../CalculatedFieldControllerTest.java | 15 ++++++++------- .../service/CalculatedFieldServiceTest.java | 18 ------------------ 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 3f505c78ef..67e7bc64bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -22,7 +22,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -123,25 +124,25 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index de6836900d..a94eb83fa3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,8 +25,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +32,6 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -148,21 +145,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } - private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - return calculatedFieldLink; - } - - private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { - CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); - calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); - return calculatedFieldLinkConfiguration; - } - private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 23009c9aafd7e1dd616773621284f33abc537616 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 17:21:03 +0200 Subject: [PATCH 020/281] refactored defaultTbCalculatedFieldService methods --- .../entitiy/AbstractTbEntityService.java | 7 + .../entitiy/cf/CalculatedFieldCtx.java | 13 +- .../cf/DefaultTbCalculatedFieldService.java | 177 +++++++----------- .../data/cf/CalculatedFieldConfiguration.java | 3 + 4 files changed, 90 insertions(+), 110 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index d2b3890c70..cef32b9065 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,18 +89,25 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired + @Lazy protected AccessControlService accessControlService; @Autowired + @Lazy protected TenantService tenantService; @Autowired + @Lazy protected AssetService assetService; @Autowired + @Lazy protected DeviceService deviceService; @Autowired + @Lazy protected AssetProfileService assetProfileService; @Autowired + @Lazy protected DeviceProfileService deviceProfileService; @Autowired + @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 61f28304f2..0a7a5c8264 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -15,17 +15,20 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @Data -@Builder public class CalculatedFieldCtx { - private final CalculatedFieldId calculatedFieldId; - private final EntityId entityId; - private final CalculatedFieldState state; + private CalculatedFieldId calculatedFieldId; + private EntityId entityId; + private CalculatedFieldState state; + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { + this.calculatedFieldId = calculatedFieldId; + this.entityId = entityId; + this.state = state; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index e222259a53..b439d8e641 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,7 +39,6 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -48,8 +48,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.attributes.AttributesService; @@ -62,6 +61,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -71,7 +71,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -128,98 +127,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { - return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); - } - - private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { - return timeseriesService.findLatest(tenantId, entityId, keys); - } - - private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { - ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); - ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); - - ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); - - return Futures.transform(combinedFuture, results -> { - List attributes = (List) results.get(0); - List timeSeries = (List) results.get(1); - - initializeState(calculatedField, attributes, timeSeries); - - return null; - }, MoreExecutors.directExecutor()); - } - - private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - - if (state != null) { - String calculation = performCalculation(state.getArguments()); - - Map updatedArguments = state.getArguments(); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); - } else { - // initial calculation - Map arguments = calculatedField.getConfiguration().getArguments(); - - Map argumentValues = arguments.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) - )); - - String calculation = performCalculation(argumentValues); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); - } - - calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); - states.put(calculatedField.getId(), calculatedFieldCtx); - } - - private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, - List attributes, List timeSeries) { - String type = argument.getType(); - String value = null; - - if ("ATTRIBUTES".equals(type)) { - value = attributes.stream() - .filter(attribute -> attribute.getKey().equals(key)) - .map(AttributeKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } else if ("TIME_SERIES".equals(type)) { - value = timeSeries.stream() - .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) - .map(TsKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } - - return value != null ? value : argument.getDefaultValue(); - } - - private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { - for (T profileId : profileIds) { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } - } - @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -232,21 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, assetIds); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, deviceIds); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -359,7 +261,72 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private String performCalculation(Map argumentValues) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to fetch data for type: {}", argument.getType(), t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + // calculation based on the previous data + String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); + + Map updatedArguments = new HashMap<>(state.getArguments()); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + calculatedFieldCtx.setState(state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { + BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); return "calculation"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index deaeebcb50..cfae2e5a0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,6 +41,9 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + @JsonIgnore + BaseCalculatedFieldConfiguration.Output getOutput(); + @JsonIgnore List getReferencedEntities(); From 88e5da7a14068bb90bd2cb11b85217842446c220 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:06:12 +0200 Subject: [PATCH 021/281] changed key for states map --- .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index b439d8e641..2fe1f65015 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,7 +90,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); @Value("${state.initFetchPackSize:50000}") @Getter @@ -139,16 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -180,7 +180,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); - states.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); } catch (Exception e) { log.trace("Failed to process calculated field delete msg: [{}]", proto, e); callback.onFailure(e); @@ -261,7 +261,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @@ -278,7 +278,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } }, calculatedFieldCallbackExecutor)); - updateOrInitializeState(calculatedField, argumentValues); + updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -296,8 +296,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); @@ -322,7 +323,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .build(); } calculatedFieldCtx.setState(state); - states.put(calculatedField.getId(), calculatedFieldCtx); + states.put(ctxId, calculatedFieldCtx); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { From 4d8b62eb21e214cbbf2940a4ded8bac678eadf3c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:55:03 +0200 Subject: [PATCH 022/281] moved logic of cf add/update/delete msg to a single method --- .../cf/DefaultTbCalculatedFieldService.java | 55 +++++++++---------- .../entitiy/cf/TbCalculatedFieldService.java | 6 +- common/proto/src/main/proto/queue.proto | 19 ++----- 3 files changed, 30 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2fe1f65015..4c83fcf7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -128,10 +128,17 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } @Override - public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + if (proto.getDeleted()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + if (proto.getUpdated()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -163,30 +170,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - @Override - public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - } catch (Exception e) { - log.trace("Failed to process calculated field update msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - @Override - public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { - try { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - } catch (Exception e) { - log.trace("Failed to process calculated field delete msg: [{}]", proto, e); - callback.onFailure(e); - } - } - @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; @@ -223,13 +206,25 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + } catch (Exception e) { + log.trace("Failed to delete calculated field.", e); + callback.onFailure(e); + } + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); // TODO: read all states(CalculatedFieldCtx) - states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } private void checkEntityExistence(TenantId tenantId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 4f14270c3d..aa77d29702 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -24,11 +24,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); - - void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); - - void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index e1d6320779..b41e010a2d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,25 +1267,14 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldAddMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldDeleteMsgProto { +message CalculatedFieldMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 calculatedFieldIdMSB = 3; int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; } /** From 3072861a8fa986cda6b57561a343fd3d1c6c094b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 17:42:55 +0200 Subject: [PATCH 023/281] implemented methods for calculated field update/delete in cluster service --- application/pom.xml | 4 + .../entitiy/EntityStateSourcingListener.java | 28 +++++- .../cf/DefaultTbCalculatedFieldService.java | 11 ++- .../service/entitiy/cf/RocksDBService.java | 95 +++++++++++++++++++ .../queue/DefaultTbClusterService.java | 34 ++++++- .../queue/DefaultTbCoreConsumerService.java | 21 +++- .../server/utils/RocksDBConfig.java | 52 ++++++++++ .../src/main/resources/thingsboard.yml | 4 + .../server/cluster/TbClusterService.java | 5 + common/proto/src/main/proto/queue.proto | 1 + .../dao/cf/BaseCalculatedFieldService.java | 7 +- pom.xml | 7 ++ 12 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java diff --git a/application/pom.xml b/application/pom.xml index 17b179c767..62d2a49908 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,6 +369,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..6c2e995b82 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -118,7 +119,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,7 +135,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -149,7 +154,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -177,7 +183,12 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - default -> {} + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); + } + default -> { + } } } @@ -247,6 +258,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4c83fcf7b5..06c87ff4d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AttributeScope; @@ -64,6 +65,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -71,6 +73,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -83,6 +86,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final CalculatedFieldService calculatedFieldService; private final AttributesService attributesService; private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; private ListeningScheduledExecutorService scheduledExecutor; private ListeningExecutorService calculatedFieldExecutor; @@ -212,6 +216,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + List statesToRemove = states.keySet().stream() + .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) + .collect(Collectors.toList()); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field.", e); callback.onFailure(e); @@ -223,7 +231,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - // TODO: read all states(CalculatedFieldCtx) + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -319,6 +327,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java new file mode 100644 index 0000000000..2ba3e9bb4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.springframework.stereotype.Service; +import org.thingsboard.server.utils.RocksDBConfig; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class RocksDBService { + + private final RocksDB db; + private final WriteOptions writeOptions; + + public RocksDBService(RocksDBConfig config) throws RocksDBException { + this.db = config.getDb(); + this.writeOptions = new WriteOptions().setSync(true); + } + + public void put(String key, String value) { + try { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + + public void delete(String key) { + try { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public void deleteAll(List keys) { + try (WriteBatch batch = new WriteBatch()) { + for (String key : keys) { + batch.delete(key.getBytes(StandardCharsets.UTF_8)); + } + db.write(writeOptions, batch); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public String get(String key) { + try { + byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); + return value != null ? new String(value, StandardCharsets.UTF_8) : null; + } catch (RocksDBException e) { + log.error("Failed to retrieve data from RocksDB", e); + return null; + } + } + + public Map getAll() { + Map map = new HashMap<>(); + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + String value = new String(iterator.value(), StandardCharsets.UTF_8); + map.put(key, value); + } + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } + return map; + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 301f8e8838..f7c8230716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -666,7 +668,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); @@ -743,4 +746,33 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { + var created = oldCalculatedField == null; + broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + + private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + builder.setAdded(added); + builder.setUpdated(updated); + builder.setDeleted(deleted); + TransportProtos.CalculatedFieldMsgProto msg = builder.build(); + pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..979c419003 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -85,6 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,6 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -175,7 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java new file mode 100644 index 0000000000..75a4dd138a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RocksDBConfig { + + @Value("${rocksdb.db_path}") + private String dbPath; + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public RocksDB getDb() throws RocksDBException { + if (db == null) { + Options options = new Options().setCreateIfMissing(true); + db = RocksDB.open(options, dbPath); + } + return db; + } + + @PreDestroy + public void close() { + if (db != null) { + db.close(); + db = null; + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..8b1aeedec8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +rocksdb: + # Rocksdb path + db_path: "${ROCKS_DB_PATH:}" + # Actor system parameters actors: system: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5112e93da7..f173005107 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -114,4 +115,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); + + void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b41e010a2d..95e4fa8601 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1512,6 +1512,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + CalculatedFieldMsgProto calculatedFieldMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index db5539eb96..8603dadcc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -29,20 +29,21 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @RequiredArgsConstructor -public class BaseCalculatedFieldService implements CalculatedFieldService { +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; @@ -60,6 +61,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, diff --git a/pom.xml b/pom.xml index a1d5f26ecf..9b835516fc 100755 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + + 9.4.0 @@ -2272,6 +2274,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From f75eaf8226cff33ad0d8f935f1750394a3475375 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 10:23:57 +0200 Subject: [PATCH 024/281] added rocksdb path --- .../entitiy/AbstractTbEntityService.java | 7 ------- .../server/utils/RocksDBConfig.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 21 +++++++++++++++++++ .../DefaultTbAlarmCommentServiceTest.java | 21 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index cef32b9065..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,25 +89,18 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired - @Lazy protected AccessControlService accessControlService; @Autowired - @Lazy protected TenantService tenantService; @Autowired - @Lazy protected AssetService assetService; @Autowired - @Lazy protected DeviceService deviceService; @Autowired - @Lazy protected AssetProfileService assetProfileService; @Autowired - @Lazy protected DeviceProfileService deviceProfileService; @Autowired - @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java index 75a4dd138a..9c3c02f472 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -25,7 +25,7 @@ import org.springframework.stereotype.Component; @Component public class RocksDBConfig { - @Value("${rocksdb.db_path}") + @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") private String dbPath; private RocksDB db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8b1aeedec8..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,7 +426,7 @@ sql: rocksdb: # Rocksdb path - db_path: "${ROCKS_DB_PATH:}" + db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" # Actor system parameters actors: diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 60a0ac3bf6..bc1726ff41 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index b36ca51416..71c295edc9 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; From 8079ef66efc3be7f54305758390c296cd6848356 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 14:44:40 +0200 Subject: [PATCH 025/281] added logs --- .../entitiy/cf/CalculatedFieldCtx.java | 3 ++ .../entitiy/cf/CalculatedFieldState.java | 1 + .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++++------- .../data/cf/CalculatedFieldConfiguration.java | 1 - .../dao/model/sql/CalculatedFieldEntity.java | 8 ++--- .../model/sql/CalculatedFieldLinkEntity.java | 4 +-- ...efaultNativeCalculatedFieldRepository.java | 4 +-- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 0a7a5c8264..021a9bbc83 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -26,6 +26,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldState state; + public CalculatedFieldCtx() { + } + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { this.calculatedFieldId = calculatedFieldId; this.entityId = entityId; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index dc07b820ea..f84f4a57c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -24,6 +24,7 @@ import java.util.Map; @Builder public class CalculatedFieldState { + // TODO: use value object(TsKv) instead of string Map arguments; String result; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 06c87ff4d1..51ff540a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -67,7 +67,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -113,7 +112,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); } @PreDestroy @@ -136,11 +135,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + //TODO: improve the check. Maybe it was renamed or just the name of the result changed. onCalculatedFieldDelete(calculatedFieldId, callback); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); @@ -150,13 +153,18 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); @@ -164,12 +172,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { - //Calculated field or entity was probably deleted while message was in queue; + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } } catch (Exception e) { - log.trace("Failed to process calculated field add msg: [{}]", proto, e); + log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); } } @@ -210,7 +220,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { calculatedFieldLinks.remove(calculatedFieldId); @@ -221,7 +230,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .collect(Collectors.toList()); rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { - log.trace("Failed to delete calculated field.", e); + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); } } @@ -231,7 +240,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -276,7 +285,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp @Override public void onFailure(Throwable t) { - log.warn("Failed to fetch data for type: {}", argument.getType(), t); + log.warn("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); @@ -327,7 +336,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index cfae2e5a0e..7e6a77f006 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,7 +41,6 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - @JsonIgnore BaseCalculatedFieldConfiguration.Output getOutput(); @JsonIgnore diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 725ddd7bef..efb262477b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -56,7 +56,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem private UUID tenantId; @Column(name = CALCULATED_FIELD_ENTITY_TYPE) - private EntityType entityType; + private String entityType; @Column(name = CALCULATED_FIELD_ENTITY_ID) private UUID entityId; @@ -88,12 +88,12 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.setUuid(calculatedField.getUuidId()); this.createdTime = calculatedField.getCreatedTime(); this.tenantId = calculatedField.getTenantId().getId(); - this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 0ac5a9bc94..b4d6677be4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -53,7 +53,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity Date: Thu, 14 Nov 2024 17:19:14 +0200 Subject: [PATCH 026/281] added logic to handle cf update msg --- .../entitiy/cf/CalculatedFieldState.java | 24 ++++--- .../cf/DefaultTbCalculatedFieldService.java | 67 ++++++++++++------- .../cf/SimpleCalculatedFieldState.java | 49 ++++++++++++++ .../CalculatedFieldControllerTest.java | 3 +- .../cf/BaseCalculatedFieldConfiguration.java | 1 + .../common/data/cf/CalculatedField.java | 12 ++-- .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../common/data/cf/CalculatedFieldType.java | 22 ++++++ .../SimpleCalculatedFieldConfiguration.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 9 +-- ...efaultNativeCalculatedFieldRepository.java | 7 +- .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 15 files changed, 160 insertions(+), 52 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index f84f4a57c2..9fb6a009fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,17 +15,25 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.Map; -@Data -@Builder -public class CalculatedFieldState { +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") +}) +public interface CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - Map arguments; - String result; + CalculatedFieldType getType(); + + void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 51ff540a64..49c64a1262 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -141,12 +142,11 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - //TODO: improve the check. Maybe it was renamed or just the name of the result changed. - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldUpdate(cf, callback); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); @@ -235,6 +235,31 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + } + } + + private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); @@ -309,39 +334,33 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + String ctxId = generateCtxId(calculatedField.getId(), entityId); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - // calculation based on the previous data - String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); - - Map updatedArguments = new HashMap<>(state.getArguments()); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); + state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); } else { - // initial calculation - String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); + CalculatedFieldState newState = createStateByType(calculatedField.getType()); + newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); } calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } - private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { - BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); - return "calculation"; + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + default -> + throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); + }; + } + + private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { + return calculatedFieldId.getId() + "_" + entityId.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..8a90893262 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class SimpleCalculatedFieldState implements CalculatedFieldState { + + // TODO: use value object(TsKv) instead of string + Map arguments = new HashMap<>(); + String result; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { + if (initialCalculation) { + // todo: perform initial calculation + this.arguments = argumentValues; + } else { + // todo: perform calculation based on previous data + this.arguments.putAll(argumentValues); + } + this.result = "result"; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 67e7bc64bd..ebda2cbf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -124,7 +125,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(null)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 3894835abc..4575e414ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -115,6 +115,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Data public static class Output { + private String name; private String type; private String expression; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index d96de37a39..ceb1222fe2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -20,15 +20,17 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -import java.io.Serializable; - @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -41,7 +43,7 @@ public class CalculatedField extends BaseData implements HasN @NoXss @Length(fieldName = "type") - private String type; + private CalculatedFieldType type; @NoXss @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") @@ -65,7 +67,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index 7e6a77f006..f733c35310 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -37,7 +37,7 @@ import java.util.UUID; public interface CalculatedFieldConfiguration { @JsonIgnore - String getType(); + CalculatedFieldType getType(); Map getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..89173b35b9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java index d635e0d82e..327f9cdc75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -33,7 +33,7 @@ public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfi } @Override - public String getType() { - return "SIMPLE"; + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index efb262477b..3c45a81cb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +91,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.tenantId = calculatedField.getTenantId().getId(); this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); - this.type = calculatedField.getType(); + this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); @@ -106,7 +107,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setCreatedTime(createdTime); calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - calculatedField.setType(type); + calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); @@ -118,8 +119,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case "SIMPLE": + switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 73f334e8f1..417a468b2c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -73,7 +74,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF UUID tenantId = (UUID) row.get("tenant_id"); EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); - String type = (String) row.get("type"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); @@ -133,9 +134,9 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(String type, JsonNode config, EntityType entityType, UUID entityId) { + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { switch (type) { - case "SIMPLE": + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 432d559598..43b51ce2ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -874,7 +875,7 @@ public class AssetServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAssetWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a94eb83fa3..d5d025a46d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -137,7 +138,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0e2d9b797b..1c5e0d8f49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -369,7 +370,7 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAsset.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 33b80c0d99..8f1f16e631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1212,7 +1213,7 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(deviceWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 7c433d47e2db2b291ebd99d2047f3c2e10887178 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 08:46:35 +0200 Subject: [PATCH 027/281] changed type value in cf data validator test --- .../service/validator/CalculatedFieldDataValidatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index 6d0eb0ad38..10df39e04b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -44,7 +45,7 @@ public class CalculatedFieldDataValidatorTest { @Test public void testUpdateNonExistingCalculatedField() { CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test"); given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); From 2fc32ee232c37cb9201e18adbebcc4d5e5ab7b24 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 12:07:41 +0200 Subject: [PATCH 028/281] removed unnecessary methods --- .../entitiy/cf/CalculatedFieldState.java | 2 + .../cf/DefaultTbCalculatedFieldService.java | 24 +++++--- .../src/main/resources/thingsboard.yml | 4 ++ .../server/dao/cf/CalculatedFieldService.java | 9 ++- .../dao/cf/BaseCalculatedFieldService.java | 55 ++++++++++++++++--- .../server/dao/cf/CalculatedFieldDao.java | 2 - .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 7 --- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 7 ++- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 9fb6a009fe..221c44b94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; @@ -32,6 +33,7 @@ import java.util.Map; }) public interface CalculatedFieldState { + @JsonIgnore CalculatedFieldType getType(); void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 49c64a1262..f2f16de844 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -96,7 +96,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - @Value("${state.initFetchPackSize:50000}") + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -145,7 +145,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldUpdate(cf, callback); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } } List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -172,12 +175,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { //Calculated field was probably deleted while message was in queue; log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); @@ -235,17 +239,20 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); callback.onSuccess(); + shouldReinit = false; } + return shouldReinit; } - private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; } @@ -255,9 +262,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; } private void fetchCalculatedFields() { @@ -344,6 +351,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } else { CalculatedFieldState newState = createStateByType(calculatedField.getType()); newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); + state = newState; } calculatedFieldCtx.setState(state); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97405f35d3..699d2012cc 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +# Calculated Field parameters +calculatedField: + initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" + rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index c12acade6c..4bb67f9f0d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -33,6 +34,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); @@ -45,13 +48,15 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - PageData findAllCalculatedFieldLinks(PageLink pageLink); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 8603dadcc9..fd669f7865 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -30,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -55,14 +58,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField) { - calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) - .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, @@ -80,6 +83,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -95,10 +105,28 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); } @Override @@ -125,6 +153,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); + } + @Override public List findAllCalculatedFieldLinks() { log.trace("Executing findAllCalculatedFieldLinks"); @@ -137,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); @@ -144,11 +186,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findAll(pageLink); } - @Override - public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); - } - @Override public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { return calculatedFieldDao.findAllByTenantId(tenantId).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 4abe02a09b..d1c9fd86bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -26,8 +26,6 @@ import java.util.List; public interface CalculatedFieldDao extends Dao { - boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); - List findAllByTenantId(TenantId tenantId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 728e19b890..34f2129bd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 1137b91947..bdc701070d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; @@ -31,7 +30,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -46,11 +44,6 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index a2f8f224c1..417b529dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,7 +23,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +49,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); From 24281b48b5f3b720a5b963322c474a9346a97f77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 17:11:09 +0200 Subject: [PATCH 029/281] removed properrty from thingsboard.yml --- application/src/main/resources/thingsboard.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 699d2012cc..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,10 +424,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -# Calculated Field parameters -calculatedField: - initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" - rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" From c39a373038c300a306dc2cf87bcfeebd898e4073 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 19 Nov 2024 17:21:25 +0200 Subject: [PATCH 030/281] added calculated field execution service --- .../cf/CalculatedFieldExecutionService.java | 25 ++ ...efaultCalculatedFieldExecutionService.java | 309 ++++++++++++++++++ .../entitiy/EntityStateSourcingListener.java | 16 +- .../entitiy/cf/CalculatedFieldCtx.java | 9 +- .../entitiy/cf/CalculatedFieldCtxId.java | 21 ++ .../entitiy/cf/CalculatedFieldResult.java | 46 +++ .../entitiy/cf/CalculatedFieldState.java | 9 +- .../cf/DefaultTbCalculatedFieldService.java | 258 --------------- .../service/entitiy/cf/RocksDBService.java | 2 + .../cf/ScriptCalculatedFieldState.java | 48 +++ .../cf/SimpleCalculatedFieldState.java | 37 ++- .../entitiy/cf/TbCalculatedFieldService.java | 4 - .../queue/DefaultTbCoreConsumerService.java | 10 +- .../CalculatedFieldControllerTest.java | 3 +- .../server/dao/cf/CalculatedFieldService.java | 2 + .../server/common/data/cf/Argument.java | 32 ++ .../cf/BaseCalculatedFieldConfiguration.java | 8 - .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 5 + .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 25 files changed, 566 insertions(+), 299 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java new file mode 100644 index 0000000000..6daa88d771 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +public interface CalculatedFieldExecutionService { + + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java new file mode 100644 index 0000000000..c0eeec02bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -0,0 +1,309 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.RocksDBService; +import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.partition.AbstractPartitionBasedService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; + + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @PostConstruct + public void init() { + super.init(); + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + @PreDestroy + public void stop() { + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + protected String getServiceName() { + return "Calculated Field Execution"; + } + + @Override + protected String getSchedulerExecutorName() { + return "calculated-field-scheduled"; + } + + @Override + protected Map>> onAddedPartitions(Set addedPartitions) { + // TODO: implementation for cluster mode + return Map.of(); + } + + @Override + protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { + // TODO: implementation for cluster mode + } + + @Override + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } + } + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } + case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + } + case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + } + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); + callback.onSuccess(); + } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field msg: [{}]", proto, e); + callback.onFailure(e); + } + } + + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + } + + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to initialize state for entity: [{}]", entityId, t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, entityId, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state == null) { + state = createStateByType(calculatedField.getType()); + } + state.initState(argumentValues); + calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + + state.performCalculation(calculatedField.getConfiguration()); + } + + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 6c2e995b82..1b541bcd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -65,6 +67,8 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; + private final CalculatedFieldService calculatedFieldService; + private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -102,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(event.getEntity(), event.getOldEntity()); + onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -241,11 +245,19 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(Object entity, Object oldEntity) { + private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; + // TODO: move verification of device type to cluster service + if (!oldDevice.getType().equals(device.getType())) { + DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); + if (cfExistsByProfile) { + // TODO: send device type updated msg to core + } + } } tbClusterService.onDeviceUpdated(device, oldDevice); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 021a9bbc83..8a5e4cdf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -22,16 +22,15 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class CalculatedFieldCtx { - private CalculatedFieldId calculatedFieldId; - private EntityId entityId; + private CalculatedFieldCtxId id; private CalculatedFieldState state; public CalculatedFieldCtx() { } - public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { - this.calculatedFieldId = calculatedFieldId; - this.entityId = entityId; + public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + this.id = id; this.state = state; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java new file mode 100644 index 0000000000..3dc0dead36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import java.util.UUID; + +public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..adb8b70e7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class CalculatedFieldResult { + + private String name; + private String type; + private AttributeScope scope; + private String value; + + public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "ATTRIBUTES"; + result.scope = scope; + result.value = value; + return result; + } + + public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "TIME_SERIES"; + result.value = value; + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 221c44b94c..9997df947f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.entitiy.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -36,6 +37,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); + default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { + return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + } + + void initState(Map argumentValues); + + CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index f2f16de844..0cc644606f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,65 +15,28 @@ */ package org.thingsboard.server.service.entitiy.cf; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -84,109 +47,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final AttributesService attributesService; - private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; - private ListeningScheduledExecutorService scheduledExecutor; - - private ListeningExecutorService calculatedFieldExecutor; - private ListeningExecutorService calculatedFieldCallbackExecutor; - - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - - @Value("10") - @Getter - private int defaultCalculatedFieldCheckIntervalInSec; - - @PostConstruct - public void init() { - // from AbstractPartitionBasedService - scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); - /// - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); - calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); - } - - @PreDestroy - public void stop() { - // from AbstractPartitionBasedService - if (scheduledExecutor != null) { - scheduledExecutor.shutdown(); - } - /// - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } - if (calculatedFieldCallbackExecutor != null) { - calculatedFieldCallbackExecutor.shutdownNow(); - } - } - - @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cf != null) { - EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); - } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { @@ -224,58 +84,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - List statesToRemove = states.keySet().stream() - .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) - .collect(Collectors.toList()); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); - } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); - } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> @@ -305,70 +113,4 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { - @Override - public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - - } - - private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { - return switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = generateCtxId(calculatedField.getId(), entityId); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); - } else { - CalculatedFieldState newState = createStateByType(calculatedField.getType()); - newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); - state = newState; - } - calculatedFieldCtx.setState(state); - - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); - } - - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - default -> - throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); - }; - } - - private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { - return calculatedFieldId.getId() + "_" + entityId.getId(); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java index 2ba3e9bb4f..2cf7aec18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.utils.RocksDBConfig; @@ -31,6 +32,7 @@ import java.util.Map; @Service @Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith'") public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..52435643cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class ScriptCalculatedFieldState implements CalculatedFieldState { + + private TbelInvokeService tbelInvokeService; + + private Map arguments = new HashMap<>(); + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + public void initState(Map argumentValues) { + + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java index 8a90893262..c004f9dd65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.entitiy.cf; import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -26,8 +28,8 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - Map arguments = new HashMap<>(); - String result; + private Map arguments = new HashMap<>(); + private String outputResult; @Override public CalculatedFieldType getType() { @@ -35,15 +37,30 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { - if (initialCalculation) { - // todo: perform initial calculation - this.arguments = argumentValues; - } else { - // todo: perform calculation based on previous data - this.arguments.putAll(argumentValues); + public void initState(Map argumentValues) { + this.arguments = argumentValues; + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + if (isValid(arguments, calculatedFieldConfiguration)) { + String expression = calculatedFieldConfiguration.getOutput().getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(arguments.keySet()) + .build(); + customExpression.set(expr); + } + Map variables = new HashMap<>(); + arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + expr.setVariables(variables); + double result = expr.evaluate(); + this.outputResult = Double.toString(result); } - this.result = "result"; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index aa77d29702..89931b8541 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,14 +18,10 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 979c419003..2d3323e097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -86,7 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -150,7 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -179,7 +179,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ebda2cbf82..57d7afcaba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -136,7 +137,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 4bb67f9f0d..e90f52c5c0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -60,4 +60,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java new file mode 100644 index 0000000000..bcd22f9216 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class Argument { + + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + + private int limit; + private long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 4575e414ac..be69b951d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -105,14 +105,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return configNode; } - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private String defaultValue; - } - @Data public static class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index f733c35310..a3598cb59d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -39,7 +39,7 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldType getType(); - Map getArguments(); + Map getArguments(); BaseCalculatedFieldConfiguration.Output getOutput(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fd669f7865..1ee881ae72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -195,6 +195,11 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByEntityId(tenantId, entityId); + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index d1c9fd86bc..a6b7c2dea1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -34,4 +34,6 @@ public interface CalculatedFieldDao extends Dao { List removeAllByEntityId(TenantId tenantId, EntityId entityId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index bdc701070d..737a089a15 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -66,6 +66,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 43b51ce2ac..2082210899 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -880,7 +881,7 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index d5d025a46d..751d883896 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -149,7 +150,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 1c5e0d8f49..73d398bb39 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -375,7 +376,7 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 8f1f16e631..1bd876eae0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -1218,7 +1219,7 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); From 40299cab1ac2a54a73c09f342b48c5923d52d99b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 20 Nov 2024 17:34:07 +0200 Subject: [PATCH 031/281] added nofitication on update in telemetry service --- .../cf/CalculatedFieldExecutionService.java | 8 +++ .../cf/CalculatedFieldResult.java | 23 ++---- ...efaultCalculatedFieldExecutionService.java | 70 +++++++++++++++---- .../{entitiy => }/cf/RocksDBService.java | 4 +- .../cf => cf/ctx}/CalculatedFieldCtx.java | 5 +- .../cf => cf/ctx}/CalculatedFieldCtxId.java | 2 +- .../ctx/state}/CalculatedFieldState.java | 14 ++-- .../state}/ScriptCalculatedFieldState.java | 19 +++-- .../state}/SimpleCalculatedFieldState.java | 38 +++++++--- .../cf/DefaultTbCalculatedFieldService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 60 +++++++++++++++- .../CalculatedFieldControllerTest.java | 9 +-- .../server/dao/cf/CalculatedFieldService.java | 4 ++ .../common/data/cf/CalculatedField.java | 2 + .../cf/CalculatedFieldLinkConfiguration.java | 8 +-- .../data/cf/{ => configuration}/Argument.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 43 ++++++------ .../CalculatedFieldConfiguration.java | 9 ++- .../common/data/cf/configuration/Output.java | 29 ++++++++ .../ScriptCalculatedFieldConfiguration.java | 40 +++++++++++ .../SimpleCalculatedFieldConfiguration.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 16 ++++- .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/model/sql/CalculatedFieldEntity.java | 7 +- .../sql/cf/CalculatedFieldLinkRepository.java | 2 + ...efaultNativeCalculatedFieldRepository.java | 4 +- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 ++ .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 9 +-- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 31 files changed, 349 insertions(+), 115 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/CalculatedFieldResult.java (53%) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/RocksDBService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtx.java (84%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtxId.java (93%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/CalculatedFieldState.java (70%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/ScriptCalculatedFieldState.java (73%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/SimpleCalculatedFieldState.java (57%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/Argument.java (85%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/BaseCalculatedFieldConfiguration.java (81%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/CalculatedFieldConfiguration.java (81%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/SimpleCalculatedFieldConfiguration.java (90%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6daa88d771..9a8b26d074 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,11 +15,19 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Map; + public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + +// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index adb8b70e7e..4982445735 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -13,34 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import java.util.Map; + @Data public class CalculatedFieldResult { - private String name; private String type; private AttributeScope scope; - private String value; - - public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "ATTRIBUTES"; - result.scope = scope; - result.value = value; - return result; - } + private Map resultMap; - public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "TIME_SERIES"; - result.value = value; - return result; + public CalculatedFieldResult() { } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0eeec02bb..250496830c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -29,13 +30,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.Argument; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -44,7 +44,10 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -54,12 +57,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.RocksDBService; -import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; @@ -71,6 +73,9 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; @TbCoreComponent @Service @@ -84,6 +89,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; + private final TbClusterService clusterService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -194,6 +200,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + try { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); boolean shouldReinit = true; @@ -250,11 +267,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); + AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); argumentValues.put(key, value); + if (remaining.decrementAndGet() == 0) { + updateOrInitializeState(calculatedField, entityId, argumentValues); + } } @Override @@ -263,15 +284,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - } private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), result -> result.map(entry -> (KvEntry) entry), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( @@ -296,7 +314,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - state.performCalculation(calculatedField.getConfiguration()); + CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + try { + String type = calculatedFieldResult.getType(); + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + } catch (Exception e) { + log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + } + } + + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { + ObjectNode jsonNodes = JacksonUtil.newObjectNode(); + calculatedFieldResult.getResultMap().forEach(jsonNodes::put); + return jsonNodes; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java rename to application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 2cf7aec18d..d6b2980042 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; @@ -94,4 +94,4 @@ public class RocksDBService { return map; } -} \ No newline at end of file +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java index 8a5e4cdf65..4b2a6c918f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import lombok.Data; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data public class CalculatedFieldCtx { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java index 3dc0dead36..a316c54b76 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 9997df947f..c25a6960ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -30,15 +31,16 @@ import java.util.Map; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") }) public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { - return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + default boolean isValid(Map argumentValues, Map arguments) { + return argumentValues.keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 52435643cc..238e8005f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -13,23 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @Data +@Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private TbelInvokeService tbelInvokeService; - private Map arguments = new HashMap<>(); + public ScriptCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -37,11 +40,15 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - + if (arguments == null) { + this.arguments = new HashMap<>(); + } + this.arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + // TODO: implement return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c004f9dd65..e984e300c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @@ -28,8 +31,7 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - private Map arguments = new HashMap<>(); - private String outputResult; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -38,29 +40,43 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - this.arguments = argumentValues; + if (arguments == null) { + arguments = new HashMap<>(); + } + arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - if (isValid(arguments, calculatedFieldConfiguration)) { - String expression = calculatedFieldConfiguration.getOutput().getExpression(); + Output output = calculatedFieldConfiguration.getOutput(); + Map arguments = calculatedFieldConfiguration.getArguments(); + + if (isValid(this.arguments, arguments)) { + CalculatedFieldResult result = new CalculatedFieldResult(); + String expression = output.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) .implicitMultiplication(true) - .variables(arguments.keySet()) + .variables(this.arguments.keySet()) .build(); customExpression.set(expr); } Map variables = new HashMap<>(); - arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); expr.setVariables(variables); - double result = expr.evaluate(); - this.outputResult = Double.toString(result); + + String expressionResult = String.valueOf(expr.evaluate()); + + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(Map.of(output.getName(), expressionResult)); + return result; } + return null; + // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 0cc644606f..4d28ff55ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 5513c41929..ac769bee9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +42,7 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,9 +50,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,6 +82,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldExecutionService calculatedFieldExecutionService; private ExecutorService tsCallBackExecutor; @@ -87,12 +94,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldService calculatedFieldService, + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldService = calculatedFieldService; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -179,6 +190,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -187,6 +199,49 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } + private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { + if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { + List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (!cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + List filteredTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .toList(); + + + Map updatedTelemetry = new HashMap<>(); + for (KvEntry telemetryEntry : filteredTelemetry) { + String key = telemetryEntry.getKey(); + if (telemetryEntry instanceof AttributeKvEntry) { + for (Map.Entry attribute : attributes.entrySet()) { + if (telemetryEntry.getKey().equals(attribute.getValue())) { + key = attribute.getKey(); + break; + } + } + } + if (telemetryEntry instanceof TsKvEntry) { + for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { + if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { + key = timeSeriesEntry.getKey(); + break; + } + } + } + updatedTelemetry.put(key, telemetryEntry.getValueAsString()); + } + + if (!updatedTelemetry.isEmpty()) { + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + } + }); + } + } + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -263,6 +318,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -270,6 +326,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -283,6 +340,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 57d7afcaba..314dc2bdba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,11 +21,12 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -144,7 +145,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e90f52c5c0..e44ff0ba22 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -54,12 +54,16 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index ceb1222fe2..e626c9d3d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 02d668a67a..c5f81cd572 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -17,13 +17,13 @@ package org.thingsboard.server.common.data.cf; import lombok.Data; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private List attributes = new ArrayList<>(); - private List timeSeries = new ArrayList<>(); + private Map attributes = new HashMap<>(); + private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java similarity index 85% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index bcd22f9216..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; @Data @@ -24,6 +25,7 @@ public class Argument { private EntityId entityId; private String key; private String type; + private AttributeScope scope; private String defaultValue; private int limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index be69b951d1..7692d792f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -60,18 +62,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Override public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - arguments.values().stream() - .filter(argument -> argument.getEntityId().equals(entityId)) - .forEach(argument -> { - switch (argument.getType()) { - case "ATTRIBUTES": - linkConfiguration.getAttributes().add(argument.getKey()); - break; - case "TIME_SERIES": - linkConfiguration.getTimeSeries().add(argument.getKey()); - break; - } - }); + + for (Map.Entry entry : arguments.entrySet()) { + Argument argument = entry.getValue(); + if (argument.getEntityId().equals(entityId)) { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); + break; + } + } + } return linkConfiguration; } @@ -93,25 +97,21 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argumentNode.put("key", argument.getKey()); argumentNode.put("type", argument.getType()); + argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); }); if (output != null) { ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); + outputNode.put("scope", String.valueOf(output.getScope())); outputNode.put("expression", output.getExpression()); } return configNode; } - @Data - public static class Output { - private String name; - private String type; - private String expression; - } - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { if (config == null || !config.isObject()) { return null; @@ -133,6 +133,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); + argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); @@ -142,7 +143,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); + output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); + output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); output.setExpression(outputNode.get("expression").asText()); this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index a3598cb59d..155015028f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -13,13 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; @@ -32,7 +34,8 @@ import java.util.UUID; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { @@ -41,7 +44,7 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - BaseCalculatedFieldConfiguration.Output getOutput(); + Output getOutput(); @JsonIgnore List getReferencedEntities(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..683e372ebc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class Output { + + private String name; + private String type; + private AttributeScope scope; + private String expression; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..a24328b4c9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public ScriptCalculatedFieldConfiguration() { + super(); + } + + public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index 327f9cdc75..af11d2f5d8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 1ee881ae72..cf5f79c5fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -173,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + @Override public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); @@ -195,6 +201,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 34f2129bd7..549db510ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,6 +30,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 3c45a81cb5..6500d2a1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,9 +24,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -122,6 +123,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem switch (CalculatedFieldType.valueOf(type)) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index 61c4026cca..d7325df8d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -25,4 +25,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 417a468b2c..eebca14b6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,11 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 417b529dc9..29492a10cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override public ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2082210899..2012090264 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,10 +30,11 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -888,7 +889,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 751d883896..2bdb1b8897 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,11 +23,12 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.cf.CalculatedFieldService; @@ -157,7 +158,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 73d398bb39..b58f739462 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,10 +31,11 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -383,7 +384,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 1bd876eae0..38bd21170a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,10 +39,11 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1226,7 +1227,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); From c75603f57c0d093d14cb7c3cf4fe187082f70ca5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 15:45:39 +0200 Subject: [PATCH 032/281] added implementation for script type --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 43 ++++++--- .../state/CalculatedFieldScriptEngine.java | 29 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 12 ++- .../CalculatedFieldTbelScriptEngine.java | 90 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 59 ++++++++++-- .../ctx/state/SimpleCalculatedFieldState.java | 22 ++--- .../DefaultTelemetrySubscriptionService.java | 47 +++++----- .../thingsboard/script/api/ScriptType.java | 2 +- 10 files changed, 248 insertions(+), 63 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 9a8b26d074..d4e0d1da84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; @@ -26,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - -// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 4982445735..87f1d08a84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -25,7 +25,7 @@ public class CalculatedFieldResult { private String type; private AttributeScope scope; - private Map resultMap; + private Map resultMap; public CalculatedFieldResult() { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 250496830c..c83bd7a71a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -90,6 +92,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; private final TbClusterService clusterService; + private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -201,7 +204,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); @@ -266,13 +269,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); + // todo: should be rewritten implementation for default value + argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -300,7 +303,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -314,10 +317,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); - if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); - } + ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + } + }, MoreExecutors.directExecutor()); + } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { @@ -325,8 +339,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + ObjectNode payload = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); @@ -334,9 +348,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { - ObjectNode jsonNodes = JacksonUtil.newObjectNode(); - calculatedFieldResult.getResultMap().forEach(jsonNodes::put); - return jsonNodes; + ObjectNode payload = JacksonUtil.newObjectNode(); + Map resultMap = calculatedFieldResult.getResultMap(); + resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class))); + return payload; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..6b54536019 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Map arguments); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index c25a6960ac..dffcf09820 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -18,9 +18,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -39,12 +43,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { + default boolean isValid(Map argumentValues, Map arguments) { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); + ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..7e8376be8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import javax.script.ScriptException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Map arguments) { + log.trace("execute script async, arguments {}", arguments); + Object[] args = new Object[arguments.size()]; + int index = 0; + for (KvEntry entry : arguments.values()) { + switch (entry.getDataType()) { + case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); + case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); + case LONG -> args[index] = entry.getLongValue().orElse(null); + case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); + default -> args[index] = entry.getValueAsString(); + } + index++; + } + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 238e8005f2..363c237df5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,10 +15,19 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -28,7 +37,10 @@ import java.util.Map; @Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private Map arguments = new HashMap<>(); + @JsonIgnore + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + private Map arguments = new HashMap<>(); public ScriptCalculatedFieldState() { } @@ -39,7 +51,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { this.arguments = new HashMap<>(); } @@ -47,9 +59,46 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - // TODO: implement - return null; + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + if (calculatedFieldScriptEngine == null) { + initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + } + + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); + + return Futures.transform(resultFuture, result -> { + Output output = calculatedFieldConfiguration.getOutput(); + Map resultMap = new HashMap<>(); + + if (result instanceof Map) { + Map map = JacksonUtil.convertValue(result, Map.class); + if (map != null) { + resultMap.putAll(map); + } + } else { + resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); + } + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return calculatedFieldResult; + }, MoreExecutors.directExecutor()); + } + + private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + calculatedFieldConfiguration.getOutput().getExpression(), + arguments.keySet().toArray(new String[0]) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e984e300c6..725da6c7a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -30,8 +35,7 @@ import java.util.Map; @Data public class SimpleCalculatedFieldState implements CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - private Map arguments; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -39,7 +43,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } @@ -47,7 +51,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); @@ -64,19 +68,17 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); expr.setVariables(variables); - String expressionResult = String.valueOf(expr.evaluate()); + double expressionResult = expr.evaluate(); result.setType(output.getType()); result.setScope(output.getScope()); result.setResultMap(Map.of(output.getName(), expressionResult)); - return result; + return Futures.immediateFuture(result); } - return null; - // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ac769bee9e..76c1383f6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -69,6 +69,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -207,32 +208,28 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); Map timeSeries = link.getConfiguration().getTimeSeries(); - List filteredTelemetry = telemetry.stream() + Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .toList(); - - - Map updatedTelemetry = new HashMap<>(); - for (KvEntry telemetryEntry : filteredTelemetry) { - String key = telemetryEntry.getKey(); - if (telemetryEntry instanceof AttributeKvEntry) { - for (Map.Entry attribute : attributes.entrySet()) { - if (telemetryEntry.getKey().equals(attribute.getValue())) { - key = attribute.getKey(); - break; - } - } - } - if (telemetryEntry instanceof TsKvEntry) { - for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { - if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { - key = timeSeriesEntry.getKey(); - break; - } - } - } - updatedTelemetry.put(key, telemetryEntry.getValueAsString()); - } + .collect(Collectors.toMap( + entry -> { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + }, + entry -> entry, + (v1, v2) -> v1 + )); if (!updatedTelemetry.isEmpty()) { calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index cdcdf815d0..7f8c513957 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } From 6ed4abb9038eb424c129965d99fae7ed828c633a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 17:39:16 +0200 Subject: [PATCH 033/281] separated expression from output --- ...efaultCalculatedFieldExecutionService.java | 4 +-- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../CalculatedFieldControllerTest.java | 8 +++--- .../BaseCalculatedFieldConfiguration.java | 27 +++++++++++++++---- .../CalculatedFieldConfiguration.java | 2 ++ .../common/data/cf/configuration/Output.java | 1 - ...efaultNativeCalculatedFieldRepository.java | 5 +++- .../server/dao/service/AssetServiceTest.java | 6 +++-- .../service/CalculatedFieldServiceTest.java | 8 +++--- .../dao/service/CustomerServiceTest.java | 6 +++-- .../server/dao/service/DeviceServiceTest.java | 6 +++-- 12 files changed, 54 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c83bd7a71a..6aa9997d1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -253,9 +253,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void fetchCalculatedFields() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 363c237df5..cbee8dc902 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -96,7 +96,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getOutput().getExpression(), + calculatedFieldConfiguration.getExpression(), arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 725da6c7a7..08c4741de5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { if (isValid(this.arguments, arguments)) { CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = output.getExpression(); + String expression = calculatedFieldConfiguration.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 314dc2bdba..77ca268d12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,10 +21,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; @@ -145,9 +145,11 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 7692d792f8..55a43a00ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -40,6 +40,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; + protected String expression; protected Output output; public BaseCalculatedFieldConfiguration() { @@ -48,6 +49,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); + this.expression = calculatedFieldConfig.getExpression(); this.output = calculatedFieldConfig.getOutput(); } @@ -101,12 +103,17 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("defaultValue", argument.getDefaultValue()); }); + if (expression != null) { + configNode.put("expression", expression); + } + if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); - outputNode.put("scope", String.valueOf(output.getScope())); - outputNode.put("expression", output.getExpression()); + if (output.getScope() != null) { + outputNode.put("scope", String.valueOf(output.getScope())); + } } return configNode; @@ -133,20 +140,30 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); - argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); + JsonNode scope = argumentNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + argument.setScope(AttributeScope.valueOf(scope.asText())); + } argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); } this.setArguments(arguments); + JsonNode expressionNode = config.get("expression"); + if (expressionNode != null && expressionNode.isTextual()) { + this.setExpression(expressionNode.asText()); + } + JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); - output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); - output.setExpression(outputNode.get("expression").asText()); + JsonNode scope = outputNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + output.setScope(AttributeScope.valueOf(scope.asText())); + } this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 155015028f..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -44,6 +44,8 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + String getExpression(); + Output getOutput(); @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 683e372ebc..46257d1ccc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -24,6 +24,5 @@ public class Output { private String name; private String type; private AttributeScope scope; - private String expression; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index eebca14b6e..fc40f72c93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,10 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -138,6 +139,8 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF switch (type) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2012090264..87f2cb1f45 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,9 +30,9 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -889,9 +889,11 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 2bdb1b8897..77ed026b1d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,10 +23,10 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -158,9 +158,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index b58f739462..94c8440057 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,9 +31,9 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; @@ -384,9 +384,11 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 38bd21170a..d5394a3494 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,9 +39,9 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -1227,9 +1227,11 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); From 31103e90e3e3cb5f440823872f2a0c85cdac6b10 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 22 Nov 2024 17:17:59 +0200 Subject: [PATCH 034/281] added implementation to handle update entity profiles events --- .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 74 ++++++++++++++++--- .../CalculatedFieldTbelScriptEngine.java | 14 +--- .../ctx/state/ScriptCalculatedFieldState.java | 13 +--- .../entitiy/EntityStateSourcingListener.java | 31 ++++---- .../queue/DefaultTbClusterService.java | 44 ++++++++++- .../queue/DefaultTbCoreConsumerService.java | 19 ++++- .../server/cluster/TbClusterService.java | 3 + .../server/dao/cf/CalculatedFieldService.java | 2 + common/proto/src/main/proto/queue.proto | 34 ++++++--- .../server/dao/asset/BaseAssetService.java | 4 +- .../dao/cf/BaseCalculatedFieldService.java | 14 +++- .../server/dao/cf/CalculatedFieldDao.java | 3 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++ 15 files changed, 202 insertions(+), 64 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d4e0d1da84..6b7b12655f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,4 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6aa9997d1f..9edea50a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -27,12 +27,14 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -44,8 +46,14 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -210,7 +218,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + + @Override + public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); + cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); + List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); + rocksDBService.deleteAll(ctxIdsToDelete); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) + .forEach(cfId -> { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + }); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) + .stream() + .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) + .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); } } @@ -271,10 +310,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - // todo: should be rewritten implementation for default value argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); @@ -289,20 +327,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of( + new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -322,7 +378,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7e8376be8e..5cc58b2a95 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -55,18 +54,7 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng @Override public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = new Object[arguments.size()]; - int index = 0; - for (KvEntry entry : arguments.values()) { - switch (entry.getDataType()) { - case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); - case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); - case LONG -> args[index] = entry.getLongValue().orElse(null); - case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); - default -> args[index] = entry.getValueAsString(); - } - index++; - } + Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index cbee8dc902..5c984a8d16 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -72,16 +72,9 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return Futures.transform(resultFuture, result -> { Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = new HashMap<>(); - - if (result instanceof Map) { - Map map = JacksonUtil.convertValue(result, Map.class); - if (map != null) { - resultMap.putAll(map); - } - } else { - resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); - } + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); calculatedFieldResult.setType(output.getType()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 1b541bcd5d..154f5f4833 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; @@ -51,8 +52,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -67,8 +66,6 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; - private final CalculatedFieldService calculatedFieldService; - private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -88,7 +85,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -106,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); + onDeviceUpdate(event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -245,23 +245,24 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { + private void onDeviceUpdate(Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; - // TODO: move verification of device type to cluster service - if (!oldDevice.getType().equals(device.getType())) { - DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); - if (cfExistsByProfile) { - // TODO: send device type updated msg to core - } - } } tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f7c8230716..58a16d9c0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,6 +68,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -149,6 +150,7 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; + private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -609,7 +611,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { + boolean deviceTypeChanged = !device.getType().equals(old.getType()); + if (deviceTypeChanged) { + handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + } + if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } @@ -618,6 +624,26 @@ public class DefaultTbClusterService implements TbClusterService { otaPackageStateService.update(device, old); } + @Override + public void onAssetUpdated(Asset asset, Asset old) { + var created = old == null; + broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + if (old != null) { + boolean assetTypeChanged = !asset.getType().equals(old.getType()); + if (assetTypeChanged) { + handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + } + } + broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + if (cfExistsByProfile) { + sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -775,4 +801,20 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } + private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2d3323e097..3caaab6613 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -157,6 +158,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; + private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -202,6 +204,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -315,6 +318,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -677,6 +682,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index f173005107..131b50a52c 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -97,6 +98,8 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e44ff0ba22..1e64fdac60 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -36,6 +36,8 @@ public interface CalculatedFieldService extends EntityDaoService { ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 95e4fa8601..bf49a68c80 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -771,6 +771,29 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message CalculatedFieldMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message EntityProfileUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 oldProfileIdMSB = 7; + int64 oldProfileIdLSB = 8; + int64 newProfileIdMSB = 9; + int64 newProfileIdLSB = 10; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1267,16 +1290,6 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - /** TB Core to Version Control Service */ @@ -1513,6 +1526,7 @@ message ToCoreMsg { DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 2135792174..7242cfde68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -177,8 +177,8 @@ public class BaseAssetService extends AbstractCachedEntityService findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -133,7 +141,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - validateId(entityId.getId(), id -> "Incorrect entityId " + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); return calculatedFields.size(); } @@ -212,7 +220,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); - }; + } @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index a6b7c2dea1..5b3bcc2750 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -28,6 +29,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 333057e8c5..9aa0aee428 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; @@ -25,6 +26,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 737a089a15..e3762f6157 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 2c7c6f0c5edfe140a1ce0e528824ec58c48b5647 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 08:57:09 +0200 Subject: [PATCH 035/281] fixed tests for cluster service --- .../server/service/queue/DefaultTbClusterServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 25fe589a08..cb61980cf4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; From c6d91c4ce870886e4af404246020dba15e089dd7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 17:28:44 +0200 Subject: [PATCH 036/281] added last records type of cf --- ...efaultCalculatedFieldExecutionService.java | 84 ++++++++++++++--- .../service/cf/ctx/state/ArgumentEntry.java | 30 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 10 +- .../cf/ctx/state/CalculationContext.java | 35 +++++++ .../LastRecordsCalculatedFieldState.java | 91 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 17 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 13 ++- .../common/data/cf/CalculatedFieldType.java | 2 +- .../data/cf/configuration/Argument.java | 1 + .../BaseCalculatedFieldConfiguration.java | 16 +++- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 ++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 16 ++-- ...efaultNativeCalculatedFieldRepository.java | 14 ++- 14 files changed, 323 insertions(+), 48 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9edea50a05..a985f093c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -48,12 +48,16 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -69,7 +73,10 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; @@ -84,6 +91,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -109,6 +117,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -215,7 +225,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntry(entry.getValue()); + if (entry.getValue() instanceof TsKvEntry) { + argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); + } + return argumentEntry; + } + )); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -308,12 +330,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { @Override - public void onSuccess(Optional result) { - argumentValues.put(key, result.orElse(null)); + public void onSuccess(ArgumentEntry result) { + argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -327,10 +349,37 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + TenantId tenantId = calculatedField.getTenantId(); + EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - return switch (argument.getType()) { + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + ? cfEntityId + : argumentEntityId; + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + return fetchLastRecords(tenantId, entityId, argument); + } + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + long startTs = Math.max(argument.getStartTs(), 0); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long endTs = startTs + timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(lastRecordsFuture, lastRecords -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntries(lastRecords); + return argumentEntry; + }, calculatedFieldExecutor); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + ListenableFuture> kvEntryFuture = switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), result -> result.or(() -> Optional.of( @@ -342,9 +391,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas result -> result.or(() -> Optional.of( new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) )), - MoreExecutors.directExecutor()); + calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; + return Futures.transform(kvEntryFuture, kvEntry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + if (kvEntry.isPresent()) { + argumentEntry.setKvEntry(kvEntry.orElse(null)); + } + return argumentEntry; + }, calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -359,7 +415,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -373,7 +429,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + CalculationContext ctx = CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .configuration(calculatedField.getConfiguration()) + .tbelInvokeService(tbelInvokeService) + .build(); + ListenableFuture resultFuture = state.performCalculation(ctx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -414,6 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); + case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..29e3417bde --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class ArgumentEntry { + + private KvEntry kvEntry; + private List kvEntries; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index dffcf09820..e4c4440921 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -19,11 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -36,7 +33,8 @@ import java.util.Map; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { @@ -47,8 +45,8 @@ public interface CalculatedFieldState { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); + ListenableFuture performCalculation(CalculationContext ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java new file mode 100644 index 0000000000..656763ea48 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +@Data +@Builder +public class CalculationContext { + + private TenantId tenantId; + private CalculatedFieldConfiguration configuration; + private TbelInvokeService tbelInvokeService; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java new file mode 100644 index 0000000000..2f26d71f91 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class LastRecordsCalculatedFieldState implements CalculatedFieldState { + + private Map> arguments; + + public LastRecordsCalculatedFieldState() { + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } + + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + argumentValues.forEach((key, argumentEntry) -> { + List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); + tsKvEntryList.addAll(argumentEntry.getKvEntries()); + }); + } + + + @Override + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration configuration = ctx.getConfiguration(); + Map configArguments = configuration.getArguments(); + Output output = configuration.getOutput(); + + Map resultMap = new HashMap<>(); + + arguments.replaceAll((key, entries) -> { + int limit = configArguments.get(key).getLimit(); + List limitedEntries = entries.stream() + .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) + .limit(limit) + .collect(Collectors.toList()); + + Map valueWithTs = limitedEntries.stream() + .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); + resultMap.put(key, valueWithTs); + + return limitedEntries; + }); + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return Futures.immediateFuture(calculatedFieldResult); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 5c984a8d16..0e9b00ad7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -40,7 +40,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments = new HashMap<>(); + private Map arguments; public ScriptCalculatedFieldState() { } @@ -50,22 +50,27 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { - this.arguments = new HashMap<>(); + arguments = new HashMap<>(); } - this.arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } + @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + if (tbelInvokeService == null) { throw new IllegalArgumentException("TBEL script engine is disabled!"); } if (calculatedFieldScriptEngine == null) { - initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); } ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 08c4741de5..d3d3b5f636 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,12 +20,10 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,21 +35,26 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { private Map arguments; + public SimpleCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 89173b35b9..63b6d8d1dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT + SIMPLE, SCRIPT, LAST_RECORDS } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..0d70591a38 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,6 +29,7 @@ public class Argument { private String defaultValue; private int limit; + private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 55a43a00ce..f7cc53b7cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -101,6 +101,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("type", argument.getType()); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); + argumentNode.put("limit", String.valueOf(argument.getLimit())); + argumentNode.put("startTs", String.valueOf(argument.getStartTs())); + argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); if (expression != null) { @@ -144,7 +147,18 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); } - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + if (argumentNode.hasNonNull("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + } + if (argumentNode.hasNonNull("limit")) { + argument.setLimit(argumentNode.get("limit").asInt()); + } + if (argumentNode.hasNonNull("startTs")) { + argument.setStartTs(argumentNode.get("startTs").asLong()); + } + if (argumentNode.hasNonNull("timeWindow")) { + argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); + } arguments.put(key, argument); }); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..15f7a82c40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,7 +35,8 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c3f5804227 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public LastRecordsCalculatedFieldConfiguration() { + } + + public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6500d2a1e7..b06676f70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,8 +24,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -120,14 +121,11 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fc40f72c93..2acd4d75c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -136,14 +137,11 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (type) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } From 80d9b22068ea44a5fc3d4bada2baa5c704d24246 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 11:31:49 +0200 Subject: [PATCH 037/281] added argument entry interface --- ...efaultCalculatedFieldExecutionService.java | 66 +++++++------------ .../service/cf/ctx/state/ArgumentEntry.java | 19 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 48 ++++++++++++++ .../cf/ctx/state/CalculationContext.java | 6 +- .../service/cf/ctx/state/KvArgumentEntry.java | 31 +++++++++ .../ctx/state/LastRecordsArgumentEntry.java | 33 ++++++++++ .../LastRecordsCalculatedFieldState.java | 21 ++---- .../ctx/state/ScriptCalculatedFieldState.java | 60 ++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 31 ++------- 9 files changed, 185 insertions(+), 130 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index a985f093c9..322443c848 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -191,19 +191,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); + initializeStateForEntity(cf, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -226,17 +226,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntry(entry.getValue()); - if (entry.getValue() instanceof TsKvEntry) { - argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); - } - return argumentEntry; - } - )); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { @@ -254,11 +244,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); - cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); - List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); - rocksDBService.deleteAll(ctxIdsToDelete); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); @@ -269,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -328,11 +313,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); @@ -349,12 +334,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedField.getTenantId(); - EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) - ? cfEntityId + ? targetEntityId : argumentEntityId; if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { return fetchLastRecords(tenantId, entityId, argument); @@ -371,11 +355,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, lastRecords -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntries(lastRecords); - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -394,13 +374,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - if (kvEntry.isPresent()) { - argumentEntry.setKvEntry(kvEntry.orElse(null)); - } - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -429,12 +403,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculationContext ctx = CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .configuration(calculatedField.getConfiguration()) - .tbelInvokeService(tbelInvokeService) - .build(); - ListenableFuture resultFuture = state.performCalculation(ctx); + ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -464,6 +433,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculationContext buildCalculationContext(CalculatedField calculatedField) { + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + return CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .arguments(configuration.getArguments()) + .output(configuration.getOutput()) + .expression(configuration.getExpression()) + .tbelInvokeService(tbelInvokeService) + .build(); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 29e3417bde..3097056d11 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,16 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -@Data -public class ArgumentEntry { +public interface ArgumentEntry { - private KvEntry kvEntry; - private List kvEntries; + Object getValue(); + + static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { + if (kvEntry instanceof TsKvEntry tsKvEntry) { + return new LastRecordsArgumentEntry(List.of(tsKvEntry)); + } else { + return new KvArgumentEntry(kvEntry); + } + } + + static ArgumentEntry createArgumentEntry(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..bac318a1b9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; + +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected Map arguments; + + public BaseCalculatedFieldState() { + } + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } +// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); + } + + protected CalculatedFieldResult buildResult(Output output, Map resultMap) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(resultMap); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java index 656763ea48..aaabaec0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Builder; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -29,7 +31,9 @@ import java.util.Map; public class CalculationContext { private TenantId tenantId; - private CalculatedFieldConfiguration configuration; + private Map arguments; + private Output output; + private String expression; private TbelInvokeService tbelInvokeService; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java new file mode 100644 index 0000000000..0bd21d452e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +public class KvArgumentEntry implements ArgumentEntry { + + private final KvEntry kvEntry; + + @Override + public Object getValue() { + return kvEntry; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java new file mode 100644 index 0000000000..8729f022fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class LastRecordsArgumentEntry implements ArgumentEntry { + + private final List kvEntries; + + @Override + public Object getValue() { + return kvEntries; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 2f26d71f91..0428d0823d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -33,7 +35,7 @@ import java.util.Map; import java.util.stream.Collectors; @Data -public class LastRecordsCalculatedFieldState implements CalculatedFieldState { +public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { private Map> arguments; @@ -53,21 +55,16 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { } argumentValues.forEach((key, argumentEntry) -> { List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); - tsKvEntryList.addAll(argumentEntry.getKvEntries()); +// tsKvEntryList.addAll(argumentEntry.getKvEntries()); }); } @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration configuration = ctx.getConfiguration(); - Map configArguments = configuration.getArguments(); - Output output = configuration.getOutput(); - Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { - int limit = configArguments.get(key).getLimit(); + int limit = ctx.getArguments().get(key).getLimit(); List limitedEntries = entries.stream() .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) .limit(limit) @@ -79,13 +76,7 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { return limitedEntries; }); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return Futures.immediateFuture(calculatedFieldResult); + return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0e9b00ad7d..047bfd0f8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -35,16 +35,11 @@ import java.util.Map; @Data @Slf4j -public class ScriptCalculatedFieldState implements CalculatedFieldState { +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments; - - public ScriptCalculatedFieldState() { - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -52,50 +47,35 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } + public ListenableFuture performCalculation(CalculationContext ctx) { + if (isValid(this.arguments, ctx.getArguments())) { + if (calculatedFieldScriptEngine == null) { + initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); + } + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + return Futures.transform(resultFuture, result -> { + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); + return buildResult(ctx.getOutput(), resultMap); + }, MoreExecutors.directExecutor()); } + return null; + } - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); + private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); } - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); - - return Futures.transform(resultFuture, result -> { - Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return calculatedFieldResult; - }, MoreExecutors.directExecutor()); - } - - private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getExpression(), - arguments.keySet().toArray(new String[0]) + expression, + this.arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d3d3b5f636..fbdd1eb354 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -31,36 +31,17 @@ import java.util.HashMap; import java.util.Map; @Data -public class SimpleCalculatedFieldState implements CalculatedFieldState { - - private Map arguments; - - public SimpleCalculatedFieldState() { - } +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - - Output output = calculatedFieldConfiguration.getOutput(); - Map arguments = calculatedFieldConfiguration.getArguments(); - - if (isValid(this.arguments, arguments)) { - CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = calculatedFieldConfiguration.getExpression(); + if (isValid(this.arguments, ctx.getArguments())) { + String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { @@ -76,10 +57,8 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { double expressionResult = expr.evaluate(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(Map.of(output.getName(), expressionResult)); - return Futures.immediateFuture(result); + Output output = ctx.getOutput(); + return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); } return null; } From d684c8777a2d7ec438e8a4b7b7de0a6841eaae49 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 17:33:01 +0200 Subject: [PATCH 038/281] improved usage of calculation ctx --- .../cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/CalculatedFieldResult.java | 8 +- ...efaultCalculatedFieldExecutionService.java | 95 +++++++++---------- ...Ctx.java => CalculatedFieldEntityCtx.java} | 8 +- ...d.java => CalculatedFieldEntityCtxId.java} | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 26 +++-- ...KvArgumentEntry.java => ArgumentType.java} | 15 +-- .../ctx/state/BaseCalculatedFieldState.java | 30 +++--- .../cf/ctx/state/CalculatedFieldCtx.java | 72 ++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 7 +- .../cf/ctx/state/CalculatedFieldState.java | 8 +- .../CalculatedFieldTbelScriptEngine.java | 18 +++- .../ctx/state/LastRecordsArgumentEntry.java | 16 +++- .../LastRecordsCalculatedFieldState.java | 46 +++++---- .../ctx/state/ScriptCalculatedFieldState.java | 47 ++------- .../ctx/state/SimpleCalculatedFieldState.java | 9 +- ...ext.java => SingleValueArgumentEntry.java} | 32 +++---- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../data/cf/configuration/Argument.java | 1 - .../BaseCalculatedFieldConfiguration.java | 4 - 20 files changed, 257 insertions(+), 191 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtx.java => CalculatedFieldEntityCtx.java} (79%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtxId.java => CalculatedFieldEntityCtxId.java} (90%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{KvArgumentEntry.java => ArgumentType.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{CalculationContext.java => SingleValueArgumentEntry.java} (51%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6b7b12655f..6d7cf0e741 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,6 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 87f1d08a84..1f8a06c8fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -21,13 +21,15 @@ import org.thingsboard.server.common.data.AttributeScope; import java.util.Map; @Data -public class CalculatedFieldResult { +public final class CalculatedFieldResult { private String type; private AttributeScope scope; private Map resultMap; - public CalculatedFieldResult() { + public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + this.type = type; + this.scope = scope; + this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 322443c848..0d269f032f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -115,7 +115,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -188,22 +189,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = cf.getEntityId(); calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); + CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(cf, entityId, callback); + initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -224,10 +227,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + return new CalculatedFieldCtx(calculatedField, tbelInvokeService); + }); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -235,7 +241,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -246,15 +252,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() - .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -267,6 +273,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -277,6 +284,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); + calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); List statesToRemove = states.keySet().stream() .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) @@ -309,20 +317,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedField, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } } @@ -334,28 +342,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedField.getTenantId(); + private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { return fetchLastRecords(tenantId, entityId, argument); } return fetchKvEntry(tenantId, entityId, argument); } private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { - long startTs = Math.max(argument.getStartTs(), 0); + long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long endTs = startTs + timeWindow; + long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -374,7 +382,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -389,32 +397,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); - CalculatedFieldState state = calculatedFieldCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); if (state == null) { - state = createStateByType(calculatedField.getType()); + state = createStateByType(calculatedFieldCtx.getCfType()); } state.initState(argumentValues); - calculatedFieldCtx.setState(state); - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); } } @Override public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } }, MoreExecutors.directExecutor()); @@ -433,17 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculationContext buildCalculationContext(CalculatedField calculatedField) { - CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - return CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .arguments(configuration.getArguments()) - .output(configuration.getOutput()) - .expression(configuration.getExpression()) - .tbelInvokeService(tbelInvokeService) - .build(); - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 4b2a6c918f..7a8384b6bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -19,15 +19,15 @@ import lombok.Data; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data -public class CalculatedFieldCtx { +public class CalculatedFieldEntityCtx { - private CalculatedFieldCtxId id; + private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldCtx() { + public CalculatedFieldEntityCtx() { } - public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index a316c54b76..f7c451efee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,5 +17,5 @@ package org.thingsboard.server.service.cf.ctx; import java.util.UUID; -public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 3097056d11..6218f38edf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,25 +15,35 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.stream.Collectors; +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") +}) public interface ArgumentEntry { + ArgumentType getType(); + Object getValue(); - static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { - if (kvEntry instanceof TsKvEntry tsKvEntry) { - return new LastRecordsArgumentEntry(List.of(tsKvEntry)); - } else { - return new KvArgumentEntry(kvEntry); - } + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry.getValue()); } - static ArgumentEntry createArgumentEntry(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries); + static ArgumentEntry createLastRecordsArgument(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index 0bd21d452e..f2f0eac60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -15,17 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; -import org.thingsboard.server.common.data.kv.KvEntry; - -@Data -public class KvArgumentEntry implements ArgumentEntry { - - private final KvEntry kvEntry; - - @Override - public Object getValue() { - return kvEntry; - } - +public enum ArgumentType { + SINGLE_VALUE, LAST_RECORDS } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bac318a1b9..54df89e757 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,34 +15,36 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { - protected Map arguments; + protected Map arguments; public BaseCalculatedFieldState() { } + @Override + public Map getArguments() { + return this.arguments; + } + @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } -// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - - protected CalculatedFieldResult buildResult(Output output, Map resultMap) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(resultMap); - return result; + arguments.putAll( + argumentValues.entrySet().stream() + .peek(entry -> { + if (entry.getValue() instanceof LastRecordsArgumentEntry) { + throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..10395a0d5d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +@Data +public class CalculatedFieldCtx { + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private Map arguments; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + arguments.keySet().toArray(new String[0]) + ); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 6b54536019..212d971398 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -16,13 +16,16 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.kv.KvEntry; import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Map arguments); + + ListenableFuture> executeToMapAsync(Map arguments); + + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index e4c4440921..f0882cbea4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -41,12 +41,14 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { - return argumentValues.keySet().containsAll(arguments.keySet()); + Map getArguments(); + + default boolean isValid(Map arguments) { + return getArguments().keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); - ListenableFuture performCalculation(CalculationContext ctx); + ListenableFuture performCalculation(CalculatedFieldCtx ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 5cc58b2a95..363eefd44e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -52,9 +53,9 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { + public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -71,6 +72,19 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture> executeToMapAsync(Map arguments) { + return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture> executeToMapTransform(Object result) { + if (result instanceof Map) { + return Futures.immediateFuture((Map) result); + } + throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]"); + } + @Override public void destroy() { tbelInvokeService.release(this.scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 8729f022fa..39da1838bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,19 +15,27 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import lombok.NoArgsConstructor; -import java.util.List; +import java.util.Map; @Data +@NoArgsConstructor +@AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private final List kvEntries; + private Map tsRecords; + + @Override + public ArgumentType getType() { + return ArgumentType.LAST_RECORDS; + } @Override public Object getValue() { - return kvEntries; + return tsRecords.values(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 0428d0823d..437707bfdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,14 +15,10 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,8 +33,6 @@ import java.util.stream.Collectors; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - private Map> arguments; - public LastRecordsCalculatedFieldState() { } @@ -47,36 +41,46 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.LAST_RECORDS; } - @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } argumentValues.forEach((key, argumentEntry) -> { - List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); -// tsKvEntryList.addAll(argumentEntry.getKvEntries()); + LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { + existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry + && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { + existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } }); } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { + arguments.replaceAll((key, argumentEntry) -> { int limit = ctx.getArguments().get(key).getLimit(); - List limitedEntries = entries.stream() - .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) - .limit(limit) - .collect(Collectors.toList()); - Map valueWithTs = limitedEntries.stream() - .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); - resultMap.put(key, valueWithTs); + // TODO: implement removing if size > limit + + +// List limitedEntries = entries.stream() +// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) +// .limit(limit) +// .collect(Collectors.toList()); +// +// Map valueWithTs = limitedEntries.stream() +// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); +// resultMap.put(key, valueWithTs); - return limitedEntries; +// return new LastRecordsArgumentEntry(limitedEntries); + return null; }); - return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 047bfd0f8c..16905930d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,68 +15,37 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; @Data @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { - @JsonIgnore - private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); - } - - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - - return Futures.transform(resultFuture, result -> { - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - return buildResult(ctx.getOutput(), resultMap); - }, MoreExecutors.directExecutor()); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Output output = ctx.getOutput(); + if (isValid(ctx.getArguments())) { + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } return null; } - private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); - } - - calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( - tenantId, - tbelInvokeService, - expression, - this.arguments.keySet().toArray(new String[0]) - ); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fbdd1eb354..64eb409d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -22,7 +22,6 @@ import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -39,8 +38,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + if (isValid(ctx.getArguments())) { String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); @@ -52,13 +51,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java similarity index 51% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index aaabaec0d4..504d213748 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; - -import java.util.Map; +import lombok.NoArgsConstructor; @Data -@Builder -public class CalculationContext { +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private Object value; + + @Override + public ArgumentType getType() { + return ArgumentType.SINGLE_VALUE; + } - private TenantId tenantId; - private Map arguments; - private Output output; - private String expression; - private TbelInvokeService tbelInvokeService; + @Override + public Object getValue() { + return value; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3caaab6613..392e2637d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -685,7 +685,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 0d70591a38..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,7 +29,6 @@ public class Argument { private String defaultValue; private int limit; - private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f7cc53b7cf..f311fb737b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -102,7 +102,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("startTs", String.valueOf(argument.getStartTs())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); @@ -153,9 +152,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("limit")) { argument.setLimit(argumentNode.get("limit").asInt()); } - if (argumentNode.hasNonNull("startTs")) { - argument.setStartTs(argumentNode.get("startTs").asLong()); - } if (argumentNode.hasNonNull("timeWindow")) { argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); } From f9db64a14d036cbd6a5ffdbe3a733b177e9d081b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 12:10:04 +0200 Subject: [PATCH 039/281] added deletion from map if ts out of timewindow --- .../service/cf/ctx/state/ArgumentEntry.java | 8 +++- .../ctx/state/LastRecordsArgumentEntry.java | 6 ++- .../LastRecordsCalculatedFieldState.java | 46 ++++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6218f38edf..6a62c0b1e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.TreeMap; import java.util.stream.Collectors; @JsonTypeInfo( @@ -34,16 +36,18 @@ import java.util.stream.Collectors; }) public interface ArgumentEntry { + @JsonIgnore ArgumentType getType(); Object getValue(); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { - return new SingleValueArgumentEntry(kvEntry.getValue()); + return new SingleValueArgumentEntry(kvEntry); } static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); + return new LastRecordsArgumentEntry(kvEntries.stream(). + collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 39da1838bc..93fabd3cb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,24 +15,26 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.Map; +import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private Map tsRecords; + private TreeMap tsRecords; @Override public ArgumentType getType() { return ArgumentType.LAST_RECORDS; } + @JsonIgnore @Override public Object getValue() { return tsRecords.values(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 437707bfdb..dd69790236 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -19,16 +19,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.TreeMap; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @@ -44,16 +41,15 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public void initState(Map argumentValues) { if (arguments == null) { - arguments = new HashMap<>(); + arguments = new TreeMap<>(); } argumentValues.forEach((key, argumentEntry) -> { LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry - && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { - existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); } }); } @@ -61,26 +57,22 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, argumentEntry) -> { - int limit = ctx.getArguments().get(key).getLimit(); - - // TODO: implement removing if size > limit - - -// List limitedEntries = entries.stream() -// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) -// .limit(limit) -// .collect(Collectors.toList()); -// -// Map valueWithTs = limitedEntries.stream() -// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); -// resultMap.put(key, valueWithTs); - -// return new LastRecordsArgumentEntry(limitedEntries); - return null; + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); + tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); + resultMap.put(key, tsRecords); }); Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } + private long calculateIntervalStart(long ts, long interval) { + return (ts / interval) * interval; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 64eb409d3a..fc97141806 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,9 +21,7 @@ import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +49,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 504d213748..e0db8c50fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,17 +15,29 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; @Data -@NoArgsConstructor -@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + private long ts; private Object value; + public SingleValueArgumentEntry() { + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry) { + this.ts = ((TsKvEntry) entry).getTs(); + } else if (entry instanceof AttributeKvEntry) { + this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + } + this.value = entry.getValue(); + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; From 2080b439a7e207ff186202c615f4cc161c010076 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 28 Nov 2024 11:28:51 +0200 Subject: [PATCH 040/281] added entity added/deleted events handling --- .../cf/CalculatedFieldExecutionService.java | 4 ++ ...efaultCalculatedFieldExecutionService.java | 46 ++++++++++++-- .../entitiy/EntityStateSourcingListener.java | 6 +- .../queue/DefaultTbClusterService.java | 61 +++++++++++++++++-- .../queue/DefaultTbCoreConsumerService.java | 27 ++++++++ .../server/cluster/TbClusterService.java | 2 + common/proto/src/main/proto/queue.proto | 13 ++++ .../server/dao/asset/BaseAssetService.java | 2 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d7cf0e741..a1202ec74b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -31,4 +31,8 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); + + void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0d269f032f..3bb9be065d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -74,8 +74,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -227,6 +227,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { + log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); @@ -247,7 +248,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) @@ -257,10 +257,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + @Override + public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); + log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + + @Override + public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 154f5f4833..57293169a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -146,7 +146,11 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 58a16d9c0a..78a1f27a55 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -388,11 +388,26 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + + private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); + if (cfExistsByProfile) { + sendEntityDeletedEvent(tenantId, entityId); + } + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -613,11 +628,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } + } else { + handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -631,16 +648,26 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } + } else { + handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); + if (cfExistsByOldProfile || cfExistsByNewProfile) { + sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + + private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + sendEntityAddedEvent(tenantId, entityId, profileId); } } @@ -801,7 +828,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -817,4 +844,26 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } + private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { + TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(profileId.getEntityType().name()); + builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); + builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + TransportProtos.EntityAddMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); + } + + private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { + TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 392e2637d1..d477c6d753 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,6 +320,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { + var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 131b50a52c..69334b774f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -100,6 +100,8 @@ public interface TbClusterService extends TbQueueClusterService { void onAssetUpdated(Asset asset, Asset old); + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bf49a68c80..b1ca9b5e56 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,6 +794,17 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } +message EntityAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 profileIdMSB = 7; + int64 profileIdLSB = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1527,6 +1538,8 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; + EntityAddMsgProto entityAddMsg = 55; + EntityDeleteMsg entityDeleteMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7242cfde68..564da127ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -239,7 +239,7 @@ public class BaseAssetService extends AbstractCachedEntityService Date: Thu, 28 Nov 2024 16:07:55 +0200 Subject: [PATCH 041/281] implemented update telemetry for entity by profile --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 61 +++++++++++++------ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index a1202ec74b..b605ee909e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 3bb9be065d..d821f632fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -225,7 +225,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { @@ -234,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 76c1383f6a..f6c19eb078 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -56,6 +58,8 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -85,6 +89,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbApiUsageStateService apiUsageStateService; private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private ExecutorService tsCallBackExecutor; @@ -97,7 +103,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldExecutionService calculatedFieldExecutionService, + TbAssetProfileCache assetProfileCache, + TbDeviceProfileCache deviceProfileCache) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; @@ -105,6 +113,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer this.apiUsageStateService = apiUsageStateService; this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.assetProfileCache = assetProfileCache; + this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,8 +211,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { + EntityId profileId; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + } + List cfLinks = new ArrayList<>(); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); @@ -211,34 +230,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - }, + entry -> getMappedKey(entry, attributes, timeSeries), entry -> entry, (v1, v2) -> v1 )); if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); } }); } } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), From b910760b05c8597b1e44853f4eb928c7560b0865 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 12:29:41 +0200 Subject: [PATCH 042/281] improved fetching cfs and states from db --- ...efaultCalculatedFieldExecutionService.java | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d821f632fe..2147e0a1b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,7 +36,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -81,7 +80,6 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +112,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -131,7 +128,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); } @PreDestroy @@ -176,7 +172,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -184,11 +180,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { @@ -229,7 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); }); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -300,14 +293,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); + calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); + calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -316,12 +309,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { - calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() - .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -331,6 +323,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -346,15 +342,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); @@ -433,7 +420,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -462,6 +449,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, null); + } + return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + } + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); From 894f4ead1b120d46809e6d03c326fa195fd69b0e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 17:31:56 +0200 Subject: [PATCH 043/281] implemented logic not to fetch tenant/customer telemetry for each field --- ...efaultCalculatedFieldExecutionService.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2147e0a1b7..732d050a22 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfig import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -80,6 +81,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -115,6 +117,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -366,6 +371,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); + if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { + return fetchFromStorage(tenantId, argumentEntityId, argument); + } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -375,6 +383,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } + private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { + List kvEntries; + if (EntityType.TENANT.equals(entityId.getEntityType())) { + kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + } else { + kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + } + for (KvEntry kvEntry : kvEntries) { + if (kvEntry.getKey().equals(argument.getKey())) { + return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); + } + } + return fetchKvEntry(tenantId, entityId, argument); + } + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -403,7 +426,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, kvEntry); + } + return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); + }, calculatedFieldExecutor); + } + + private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { + if (kvEntry.isEmpty()) { + return; + } + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { + kvEntries.add(kvEntry.get()); + } } private KvEntry createDefaultKvEntry(Argument argument) { From 19234df2b8e03d8d2af392faa32dfb44cc71bc77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 5 Dec 2024 16:46:12 +0200 Subject: [PATCH 044/281] improved implementation of telemetry update --- ...efaultCalculatedFieldExecutionService.java | 60 ++++++++++++------- .../DefaultTelemetrySubscriptionService.java | 11 ++-- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 732d050a22..4a1e987687 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -177,7 +177,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -226,14 +226,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); - return new CalculatedFieldCtx(calculatedField, tbelInvokeService); - }); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, Optional.of(entry.getValue())); + } + return ArgumentEntry.createSingleValueArgument(entry.getValue()); + })); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE + ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) + : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); + entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); } @@ -299,7 +317,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -328,7 +346,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } @@ -435,17 +453,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - if (kvEntry.isEmpty()) { - return; - } - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { - kvEntries.add(kvEntry.get()); - } + kvEntry.ifPresent(entry -> { + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null) { + kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); + kvEntries.add(entry); + } + }); } private KvEntry createDefaultKvEntry(Argument argument) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f6c19eb078..ad64204e0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -212,16 +212,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { - EntityId profileId; + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; if (EntityType.ASSET.equals(entityType)) { profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else { + } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); From dfeb0a7dbaf30b28529e27449f0fac9015f066f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 6 Dec 2024 11:11:24 +0200 Subject: [PATCH 045/281] made executeScrioptAsync method flexible --- .../cf/ctx/state/CalculatedFieldScriptEngine.java | 4 ++-- .../cf/ctx/state/CalculatedFieldTbelScriptEngine.java | 11 ++++------- .../cf/ctx/state/ScriptCalculatedFieldState.java | 3 ++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 212d971398..779f52c5d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -21,9 +21,9 @@ import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Object[] args); - ListenableFuture> executeToMapAsync(Map arguments); + ListenableFuture> executeToMapAsync(Object[] args); ListenableFuture> executeToMapTransform(Object result); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 363eefd44e..7ac032573f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,11 +19,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import javax.script.ScriptException; import java.util.Map; @@ -53,9 +51,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { - log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -73,8 +70,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture> executeToMapAsync(Map arguments) { - return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + public ListenableFuture> executeToMapAsync(Object[] args) { + return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 16905930d3..99befa6e65 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,7 +39,8 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From 303851015035cc3715190b5fc708ad6da173e807 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 11:04:05 +0200 Subject: [PATCH 046/281] added implementation for handling profile entities --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 96 ++++++++++--------- .../queue/DefaultTbClusterService.java | 22 ++--- .../queue/DefaultTbCoreConsumerService.java | 27 ++---- common/proto/src/main/proto/queue.proto | 7 +- 5 files changed, 71 insertions(+), 85 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index b605ee909e..5a85529f6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -32,8 +32,6 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); - void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); - - void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 4a1e987687..7ded80d013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,11 +39,9 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -83,6 +81,7 @@ import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,6 +116,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); @@ -194,17 +194,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); + case ASSET_PROFILE, DEVICE_PROFILE -> { + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -241,10 +233,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE - ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) - : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); - entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } @@ -266,6 +255,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(oldProfileId).remove(entityId); + profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -280,39 +272,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - @Override - public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { - try { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(profileId).remove(entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); + log.trace("Failed to process profile entity msg: [{}]", proto, e); } } @@ -350,6 +331,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } + private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { + return switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -365,6 +364,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 78a1f27a55..a706f943c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -404,7 +404,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityDeletedEvent(tenantId, entityId); + sendProfileEntityEvent(tenantId, entityId, profileId, false, true); } } @@ -667,7 +667,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityAddedEvent(tenantId, entityId, profileId); + sendProfileEntityEvent(tenantId, entityId, profileId, true, false); } } @@ -844,8 +844,8 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { - TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -854,16 +854,10 @@ public class DefaultTbClusterService implements TbClusterService { builder.setEntityProfileType(profileId.getEntityType().name()); builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); - TransportProtos.EntityAddMsgProto msg = builder.build(); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); - } - - private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { - TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + builder.setAdded(added); + builder.setDeleted(deleted); + TransportProtos.ProfileEntityMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d477c6d753..6cf42cb893 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,10 +320,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { + var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { - var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b1ca9b5e56..76da257b32 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,7 +794,7 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } -message EntityAddMsgProto { +message ProfileEntityMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -803,6 +803,8 @@ message EntityAddMsgProto { string entityProfileType = 6; int64 profileIdMSB = 7; int64 profileIdLSB = 8; + bool added = 9; + bool deleted = 10; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1538,8 +1540,7 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - EntityAddMsgProto entityAddMsg = 55; - EntityDeleteMsg entityDeleteMsg = 56; + ProfileEntityMsgProto profileEntityMsg = 55; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 4fed2cd416b13b7f05159ba893f47dcd788fb13d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 13:04:41 +0200 Subject: [PATCH 047/281] improved ability to fetch common arguments for profile cfs --- ...efaultCalculatedFieldExecutionService.java | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7ded80d013..f2629381b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -89,7 +88,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -117,8 +116,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); - private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -196,7 +193,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); + fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { + initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + }); + }); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -221,12 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, Optional.of(entry.getValue())); - } - return ArgumentEntry.createSingleValueArgument(entry.getValue()); - })); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); EntityId cfEntityId = calculatedField.getEntityId(); switch (cfEntityId.getEntityType()) { @@ -371,33 +367,71 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - Map arguments = calculatedFieldCtx.getArguments(); + private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { Map argumentValues = new HashMap<>(); - AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && + !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { + futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override - public void onSuccess(ArgumentEntry result) { - argumentValues.put(key, result); - if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + public void onSuccess(List results) { + onComplete.accept(argumentValues); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to fetch common arguments", t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { + Map argumentValues = new HashMap<>(commonArguments); + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!commonArguments.containsKey(key)) { + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { + @Override + public void onSuccess(List results) { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + callback.onSuccess(); } @Override public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); + log.error("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } - }, calculatedFieldCallbackExecutor)); + }, calculatedFieldCallbackExecutor); } private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); - if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { - return fetchFromStorage(tenantId, argumentEntityId, argument); - } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -407,21 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { - List kvEntries; - if (EntityType.TENANT.equals(entityId.getEntityType())) { - kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - } else { - kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - } - for (KvEntry kvEntry : kvEntries) { - if (kvEntry.getKey().equals(argument.getKey())) { - return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); - } - } - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -450,26 +469,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, kvEntry); - } - return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); - }, calculatedFieldExecutor); - } - - private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - kvEntry.ifPresent(entry -> { - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null) { - kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); - kvEntries.add(entry); - } - }); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { From 86569c312e96e297ad1de55b31c681bf32c36a28 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 10 Dec 2024 12:21:13 +0200 Subject: [PATCH 048/281] added ability to perform calculations on the last records --- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../ctx/state/LastRecordsArgumentEntry.java | 2 +- .../LastRecordsCalculatedFieldState.java | 37 ++++++++++--------- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 10395a0d5d..b436e0421e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -51,7 +51,7 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 93fabd3cb8..8b672d4de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -37,7 +37,7 @@ public class LastRecordsArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords.values(); + return tsRecords; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index dd69790236..fbcf72fda9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -56,23 +56,24 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Map resultMap = new HashMap<>(); - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); - tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); - resultMap.put(key, tsRecords); - }); - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); - } - - private long calculateIntervalStart(long ts, long interval) { - return (ts / interval) * interval; + if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 99befa6e65..ba050d3b71 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -37,10 +37,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From a73affea23683c902a5ee45ad3b569c0463fd41b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Dec 2024 17:04:24 +0200 Subject: [PATCH 049/281] removed CF type LAST_RECORDS and implemented this functionality as a script(added new argument type TS_ROLLING) --- ...efaultCalculatedFieldExecutionService.java | 49 ++++++------ .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../service/cf/ctx/state/ArgumentType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 26 +++--- .../cf/ctx/state/CalculatedFieldState.java | 2 - .../LastRecordsCalculatedFieldState.java | 79 ------------------- .../ctx/state/ScriptCalculatedFieldState.java | 14 +++- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- ...Entry.java => TsRollingArgumentEntry.java} | 4 +- .../common/data/cf/CalculatedFieldType.java | 2 +- .../BaseCalculatedFieldConfiguration.java | 4 +- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 --------- .../dao/model/sql/CalculatedFieldEntity.java | 2 - ...efaultNativeCalculatedFieldRepository.java | 2 - 15 files changed, 64 insertions(+), 172 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{LastRecordsArgumentEntry.java => TsRollingArgumentEntry.java} (91%) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2629381b2..050c2d7e73 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -73,12 +73,12 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -435,41 +435,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { - return fetchLastRecords(tenantId, entityId, argument); - } return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - ListenableFuture> kvEntryFuture = switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of( - new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - calculatedFieldExecutor); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -547,7 +547,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); - case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6a62c0b1e3..f70d614123 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), - @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") }) public interface ArgumentEntry { @@ -45,8 +45,8 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } - static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream(). + static ArgumentEntry createTsRollingArgument(List kvEntries) { + return new TsRollingArgumentEntry(kvEntries.stream(). collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index f2f0eac60d..360529a7e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentType { - SINGLE_VALUE, LAST_RECORDS + SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 54df89e757..59b007a420 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -36,15 +35,22 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll( - argumentValues.entrySet().stream() - .peek(entry -> { - if (entry.getValue() instanceof LastRecordsArgumentEntry) { - throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); - } - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + argumentValues.forEach((key, argumentEntry) -> { + ArgumentEntry existingArgumentEntry = arguments.get(key); + if (existingArgumentEntry != null) { + if (existingArgumentEntry instanceof SingleValueArgumentEntry) { + arguments.put(key, argumentEntry); + } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + } + } + } else { + arguments.put(key, argumentEntry); + } + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f0882cbea4..a5ac6b2c47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -34,7 +33,6 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java deleted file mode 100644 index fbcf72fda9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - -import java.util.Map; -import java.util.TreeMap; - -@Data -public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - - public LastRecordsCalculatedFieldState() { - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } - - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new TreeMap<>(); - } - argumentValues.forEach((key, argumentEntry) -> { - LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); - if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { - existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - } - }); - } - - @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return null; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index ba050d3b71..b9b98f9c5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; +import java.util.TreeMap; @Data @Slf4j @@ -38,6 +40,16 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); @@ -46,7 +58,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { MoreExecutors.directExecutor() ); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fc97141806..491419b40a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 8b672d4de9..1166da113d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,13 +25,13 @@ import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor -public class LastRecordsArgumentEntry implements ArgumentEntry { +public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords; @Override public ArgumentType getType() { - return ArgumentType.LAST_RECORDS; + return ArgumentType.TS_ROLLING; } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 63b6d8d1dd..89173b35b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT, LAST_RECORDS + SIMPLE, SCRIPT } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f311fb737b..ac36991a61 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -69,10 +69,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel Argument argument = entry.getValue(); if (argument.getEntityId().equals(entityId)) { switch (argument.getType()) { - case "ATTRIBUTES": + case "ATTRIBUTE": linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); break; - case "TIME_SERIES": + case "TS_LATEST", "TS_ROLLING": linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); break; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 15f7a82c40..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,8 +35,7 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java deleted file mode 100644 index c3f5804227..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; - -import java.util.UUID; - -@Data -public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - - public LastRecordsCalculatedFieldConfiguration() { - } - - public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index b06676f70b..6aaaf05836 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -124,7 +123,6 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return switch (CalculatedFieldType.valueOf(type)) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 2acd4d75c6..a5a2743f26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -140,7 +139,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF return switch (type) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } From 4668bece03229c34aa74c335bd6602ddf9885c71 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Dec 2024 11:14:56 +0200 Subject: [PATCH 050/281] fixed tests --- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../org/thingsboard/server/dao/service/AssetServiceTest.java | 4 ++-- .../server/dao/service/CalculatedFieldServiceTest.java | 4 ++-- .../thingsboard/server/dao/service/CustomerServiceTest.java | 4 ++-- .../org/thingsboard/server/dao/service/DeviceServiceTest.java | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 77ca268d12..ba1dfb1fec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -140,7 +140,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +149,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 87f2cb1f45..b0870f3dc5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -884,7 +884,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +893,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 77ed026b1d..9a1719e715 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -153,7 +153,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +162,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 94c8440057..6671e0e821 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -379,7 +379,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +388,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index d5394a3494..f2f8686bc3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1222,7 +1222,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1231,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); From 19f6f323260347e5af7eb0317dbe168dd9cb8e52 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Dec 2024 12:47:55 +0200 Subject: [PATCH 051/281] moved logic when telemetry update to main cf service --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 149 +++++++++++++----- .../cf/ctx/CalculatedFieldEntityCtx.java | 5 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 + .../ctx/state/BaseCalculatedFieldState.java | 22 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 7 +- .../ctx/state/ScriptCalculatedFieldState.java | 35 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 37 ++--- .../ctx/state/SingleValueArgumentEntry.java | 9 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 + .../DefaultTelemetrySubscriptionService.java | 78 +-------- 13 files changed, 186 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 5a85529f6b..302e4b2511 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,19 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.Map; +import java.util.List; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 1f8a06c8fa..e8ea318bf6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -23,9 +23,9 @@ import java.util.Map; @Data public final class CalculatedFieldResult { - private String type; - private AttributeScope scope; - private Map resultMap; + private final String type; + private final AttributeScope scope; + private final Map resultMap; public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { this.type = type; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 050c2d7e73..c1a86ac36f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,16 +36,20 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -76,6 +80,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +108,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -112,6 +120,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -130,6 +139,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy @@ -216,30 +235,77 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { try { - log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else if (EntityType.DEVICE.equals(entityType)) { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + Map updatedTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, attributes, timeSeries), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + } + }); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + } + } + + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); + } + + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } + return entry.getKey(); } @Override @@ -493,26 +559,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (state == null) { state = createStateByType(calculatedFieldCtx.getCfType()); } - state.initState(argumentValues); - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); - } - } + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + + boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); + if (allArgsPresent) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + } + }, MoreExecutors.directExecutor()); } - }, MoreExecutors.directExecutor()); - + } } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { @@ -520,14 +589,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (stateStr == null) { return new CalculatedFieldEntityCtx(entityCtxId, null); } - return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 7a8384b6bf..e6dc021951 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -16,17 +16,16 @@ package org.thingsboard.server.service.cf.ctx; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data +@NoArgsConstructor public class CalculatedFieldEntityCtx { private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldEntityCtx() { - } - public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f70d614123..78222244c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -41,6 +41,8 @@ public interface ArgumentEntry { Object getValue(); + boolean hasUpdatedValue(ArgumentEntry entry); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 59b007a420..ae6fc9033a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -31,26 +32,39 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public boolean updateState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } + AtomicBoolean stateUpdated = new AtomicBoolean(false); argumentValues.forEach((key, argumentEntry) -> { ArgumentEntry existingArgumentEntry = arguments.get(key); if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - arguments.put(key, argumentEntry); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + arguments.put(key, argumentEntry); + stateUpdated.set(true); + } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + stateUpdated.set(true); + } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + stateUpdated.set(true); + } + } } } else { arguments.put(key, argumentEntry); + stateUpdated.set(true); } }); + return stateUpdated.get(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index b436e0421e..2cd5c68144 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,7 +35,7 @@ public class CalculatedFieldCtx { private TenantId tenantId; private EntityId entityId; private CalculatedFieldType cfType; - private Map arguments; + private final Map arguments; private Output output; private String expression; private TbelInvokeService tbelInvokeService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index a5ac6b2c47..3c4a680df1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -41,11 +40,7 @@ public interface CalculatedFieldState { Map getArguments(); - default boolean isValid(Map arguments) { - return getArguments().keySet().containsAll(arguments.keySet()); - } - - void initState(Map argumentValues); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index b9b98f9c5e..87429050de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,26 +39,23 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); } - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return Futures.immediateFuture(null); + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 491419b40a..e16d310b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -37,27 +37,24 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); - - double expressionResult = expr.evaluate(); - - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); + String expression = ctx.getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build(); + customExpression.set(expr); } - return Futures.immediateFuture(null); + Map variables = new HashMap<>(); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); + expr.setVariables(variables); + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e0db8c50fb..e6cb24b970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -16,19 +16,18 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @Data +@NoArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - public SingleValueArgumentEntry() { - } - public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry) { this.ts = ((TsKvEntry) entry).getTs(); @@ -48,4 +47,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return this.ts != ((SingleValueArgumentEntry) entry).getTs(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1166da113d..104d0ae90c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -40,4 +40,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return tsRecords; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ad64204e0f..b94b319e71 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,11 +32,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -44,7 +40,6 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -52,14 +47,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -73,7 +65,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -87,11 +78,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private ExecutorService tsCallBackExecutor; @Value("${sql.ts.value_no_xss_validation:false}") @@ -102,19 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService, - TbAssetProfileCache assetProfileCache, - TbDeviceProfileCache deviceProfileCache) { + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; - this.assetProfileCache = assetProfileCache; - this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,7 +182,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -210,55 +191,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } - private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); - if (!cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); - Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); - } - }); - } - } - } - - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -335,7 +267,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -343,7 +275,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -357,7 +289,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } @Override From 481753b8f0e0b15f3174f65ee18fca33eb0a7568 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 17 Dec 2024 13:41:23 +0200 Subject: [PATCH 052/281] added restriction for number of cfs/arguments/ts rolling values --- .../service/cf/ctx/state/ArgumentEntry.java | 3 ++ .../ctx/state/BaseCalculatedFieldState.java | 9 ++--- .../ctx/state/SingleValueArgumentEntry.java | 8 ++++ .../cf/ctx/state/TsRollingArgumentEntry.java | 37 +++++++++++++++++-- .../cf/DefaultTbCalculatedFieldService.java | 18 +++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 78222244c9..ba7e094f77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -52,4 +52,7 @@ public interface ArgumentEntry { collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } + @JsonIgnore + ArgumentEntry copy(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ae6fc9033a..b73ac51798 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -42,25 +42,24 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); stateUpdated.set(true); } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); stateUpdated.set(true); } - } } } else { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e6cb24b970..81a57580db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -23,6 +24,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @Data @NoArgsConstructor +@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; @@ -51,4 +53,10 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return this.ts != ((SingleValueArgumentEntry) entry).getTs(); } + + @Override + public ArgumentEntry copy() { + return new SingleValueArgumentEntry(this.ts, this.value); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 104d0ae90c..49aae15ac1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,18 +16,26 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor -@AllArgsConstructor +@Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { - private TreeMap tsRecords; + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; + + private TreeMap tsRecords = new TreeMap<>(); + + public TsRollingArgumentEntry(TreeMap tsRecords) { + addAllTsRecords(tsRecords); + } @Override public ArgumentType getType() { @@ -44,4 +52,27 @@ public class TsRollingArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); } + + @Override + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + } + + public void addTsRecord(Long key, Object value) { + if (NumberUtils.isParsable(value.toString())) { + tsRecords.put(key, value); + if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { + tsRecords.pollFirstEntry(); + } + } else { + log.warn("Argument type 'TS_ROLLING' only supports numeric values."); + } + } + + public void addAllTsRecords(Map newRecords) { + for (Map.Entry entry : newRecords.entrySet()) { + addTsRecord(entry.getKey(), entry.getValue()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4d28ff55ac..2e6e975636 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -46,6 +46,9 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + private static final int MAX_ARGUMENT_SIZE = 10; + private static final int MAX_CALCULATED_FIELD_NUMBER = 10; + private final CalculatedFieldService calculatedFieldService; @Override @@ -53,7 +56,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -105,6 +110,19 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } + private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { + if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { + throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); + } + } + + private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { + int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); + if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { + throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); + } + } + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); From f929de42097c4929cae2d8970ea45dfced2b12b0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 18 Dec 2024 13:12:39 +0200 Subject: [PATCH 053/281] onAddedPartitions() impl --- ...efaultCalculatedFieldExecutionService.java | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c1a86ac36f..7e5ea5b5ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -63,6 +64,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -173,13 +175,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { - // TODO: implementation for cluster mode - return Map.of(); + var result = new HashMap>>(); + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + Map> tpiCalculatedFieldMap = new HashMap<>(); + + for (CalculatedField cf : cfs) { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + cf.getId(), cf.getTenantId(), e.getMessage()); + continue; + } + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + } + } + + for (var entry : tpiCalculatedFieldMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { + log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); + var future = calculatedFieldExecutor.submit(() -> { + try { + for (CalculatedField cf : partition) { + if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + .forEach(entityId -> restoreState(cf, entityId)); + } else { + restoreState(cf, cf.getEntityId()); + } + } + } catch (Throwable t) { + log.error("Unexpected exception while restoring CalculatedField states", t); + throw t; + } + }); + result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); + } + } + return result; + } + + private void restoreState(CalculatedField cf, EntityId entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + + if (storedState != null) { + CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); + states.put(ctxId, restoredCtx); + log.info("Restored state for CalculatedField [{}]", cf.getId()); + } else { + log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + } } @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - // TODO: implementation for cluster mode + calculatedFields.remove(entityId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); } @Override @@ -213,8 +268,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { - initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); }); }); } From 5c5dc474cbc2ba899e9aa61106a211981e122dec Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 19 Dec 2024 17:12:59 +0200 Subject: [PATCH 054/281] implemented partitioning --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++--- .../queue/DefaultTbCoreConsumerService.java | 16 ++- common/proto/src/main/proto/queue.proto | 12 ++ 4 files changed, 125 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 302e4b2511..966b5d7277 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,7 +29,9 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); - void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); + + void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7e5ea5b5ec..f2a09d0faf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -98,6 +98,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -184,7 +185,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", cf.getId(), cf.getTenantId(), e.getMessage()); continue; } @@ -233,8 +234,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - calculatedFields.remove(entityId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); + cleanupEntity(entityId); + } + + private void cleanupEntity(CalculatedFieldId calculatedFieldId) { + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @Override @@ -245,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); @@ -364,7 +369,34 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + String state = proto.getState(); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + if (calculatedFieldEntityCtx != null) { + states.put(ctxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + } else { + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + } else { + log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); + } + } catch (Exception e) { + log.trace("Failed to process calculated field update state msg: [{}]", proto, e); + } + } + + @Override + public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -377,9 +409,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } }); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); @@ -398,12 +435,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.get(profileId).remove(entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + List calculatedFieldIds = Stream.concat( + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + ).toList(); + calculatedFieldIds.forEach(cfId -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } + }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); @@ -414,11 +461,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + if (calculatedFieldState != null) { + msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + } + clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); @@ -428,8 +490,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); @@ -606,7 +674,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -616,8 +685,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); if (allArgsPresent) { @@ -626,7 +701,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(tenantId, entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6cf42cb893..a686184b8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -322,6 +322,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -708,6 +710,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 76da257b32..153394525d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -807,6 +807,17 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message CalculatedFieldStateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; + string state = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1541,6 +1552,7 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; + CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 2a41c8b45123e43fc34def1a954e098298c7339e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 20 Dec 2024 16:16:58 +0200 Subject: [PATCH 055/281] implemented logic to fetch telemetry if states were not fetched --- ...efaultCalculatedFieldExecutionService.java | 171 ++++++++++-------- .../ctx/state/SingleValueArgumentEntry.java | 10 + .../cf/ctx/state/TsRollingArgumentEntry.java | 13 +- 3 files changed, 118 insertions(+), 76 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2a09d0faf..1128a75008 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -81,12 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -97,6 +98,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,11 +202,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas var future = calculatedFieldExecutor.submit(() -> { try { for (CalculatedField cf : partition) { - if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { - restoreState(cf, cf.getEntityId()); + restoreState(cf, cfEntityId); } } } catch (Throwable t) { @@ -272,9 +275,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); } @@ -473,7 +479,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldState != null) { msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); } - clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { @@ -556,35 +562,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && - !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { - futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { - onComplete.accept(argumentValues); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to fetch common arguments", t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); } @@ -595,7 +572,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCtx.getArguments().forEach((key, argument) -> { if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), result -> { argumentValues.put(key, result); return result; @@ -618,10 +595,25 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); @@ -654,11 +646,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -676,48 +674,67 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Predicate> allArgsPresent = (args) -> + args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } + + if (allArgsPresent.test(state.getArguments())) { + performCalculation(calculatedFieldCtx, state, entityId); + } + } + }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - if (state == null) { - state = createStateByType(calculatedFieldCtx.getCfType()); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; } - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); - if (tpi.isMyPartition()) { - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); - } + performUpdateState.accept(state); + } - boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); - if (allArgsPresent) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(tenantId, entityId, result); - } - } + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } - } + }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, null); + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } @@ -725,8 +742,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); @@ -749,4 +766,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private boolean isProfileEntity(EntityId entityId) { + return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 81a57580db..3f4fd5bdce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); + private long ts; private Object value; @@ -39,6 +41,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.value = entry.getValue(); } + /** + * Internal constructor to create immutable SingleValueArgumentEntry.EMPTY + * */ + private SingleValueArgumentEntry(int ignored) { + this.ts = System.currentTimeMillis(); + this.value = null; + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 49aae15ac1..4ffa391550 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -29,6 +29,8 @@ import java.util.TreeMap; @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; private TreeMap tsRecords = new TreeMap<>(); @@ -37,6 +39,13 @@ public class TsRollingArgumentEntry implements ArgumentEntry { addAllTsRecords(tsRecords); } + /** + * Internal constructor to create immutable TsRollingArgumentEntry.EMPTY + */ + private TsRollingArgumentEntry(int ignored) { + this.tsRecords = new TreeMap<>(); + } + @Override public ArgumentType getType() { return ArgumentType.TS_ROLLING; @@ -50,7 +59,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @Override public boolean hasUpdatedValue(ArgumentEntry entry) { - return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + return entry instanceof SingleValueArgumentEntry ? + !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : + !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); } @Override From 2d65a6c457182e2f7412f083b1be7069c47d7f2d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 12:20:58 +0200 Subject: [PATCH 056/281] removed wildcard imports usage --- .../server/controller/BaseController.java | 62 ++++++++++++++++++- .../entitiy/EntityStateSourcingListener.java | 9 ++- .../DefaultTelemetrySubscriptionService.java | 17 +++-- .../rule/engine/util/TenantIdLoader.java | 33 +++++++++- .../rule/engine/util/TenantIdLoaderTest.java | 27 +++++++- 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 2d990d708d..9a980d009d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -42,7 +42,27 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HomeDashboardInfo; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -57,7 +77,37 @@ import org.thingsboard.server.common.data.edge.EdgeInfo; import org.thingsboard.server.common.data.exception.EntityVersionMismatchException; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmCommentId; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; @@ -140,7 +190,13 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 83ce48c24e..ab8246ccf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -23,7 +23,14 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf66a9f868..2ddfd22db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -29,7 +29,11 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -49,7 +53,13 @@ import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -231,8 +241,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) { - } + public void onSuccess(@Nullable Void tmp) {} @Override public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 4311d912e4..445205e88e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -19,7 +19,38 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.rule.RuleNode; import java.util.UUID; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index a879c0e3a2..921b9e26d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -23,8 +23,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.rule.engine.api.*; -import org.thingsboard.server.common.data.*; +import org.thingsboard.rule.engine.api.RuleEngineAlarmService; +import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; +import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -32,7 +47,13 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.NotificationId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.NotificationRequest; From 1482d1c4bb9983f18ab9d6abb5405f1a68496ec9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 17:00:10 +0200 Subject: [PATCH 057/281] added logic to avoid looping --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 57 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 4 +- .../thingsboard/server/common/msg/TbMsg.java | 30 ++++++++-- common/message/src/main/proto/tbmsg.proto | 7 +++ .../engine/api/AttributesSaveRequest.java | 10 +++- .../engine/api/TimeseriesSaveRequest.java | 10 +++- .../engine/telemetry/TbMsgTimeseriesNode.java | 1 + 8 files changed, 76 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 966b5d7277..ee9b6d115c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 1128a75008..e5c119bba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,11 +18,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.*; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -41,25 +37,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -301,7 +280,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { EntityType entityType = entityId.getEntityType(); if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { @@ -326,7 +305,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); } }); } @@ -335,7 +314,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); @@ -347,12 +326,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @@ -583,7 +562,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); callback.onSuccess(); } @@ -671,7 +650,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); @@ -693,7 +672,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId); + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } }; @@ -714,13 +693,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performUpdateState.accept(state); } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); } } @@ -739,13 +718,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } - private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); + if (calculatedFieldIds.contains(calculatedFieldId)) { + throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); + } + calculatedFieldIds.add(calculatedFieldId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 2ddfd22db3..e35d3cead6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -152,7 +152,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); return saveFuture; } @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); } @Override diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 64e05770fe..993fa24ff3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -64,6 +66,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List calculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +116,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +143,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.calculatedFieldIds = calculatedFieldIds; this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -200,6 +205,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new ArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +220,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +238,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -343,10 +357,12 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List calculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() {} + TbMsgBuilder() { + } TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; @@ -363,6 +379,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.calculatedFieldIds = tbMsg.calculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -454,6 +471,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +487,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } public String toString() { diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index fc9265aa14..36ab09f8d4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 22fa8de6de..9747a6033e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -22,6 +22,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -53,6 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private List calculatedFieldIds; private FutureCallback callback; Builder() {} @@ -100,6 +103,11 @@ public class AttributesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 2b5881212d..12afa2d939 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; + private List calculatedFieldIds; Builder() {} @@ -103,6 +106,11 @@ public class TimeseriesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -123,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..386e56320a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,6 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From a9f39e4917d0da31bdc89a45a8c5df344eec9d1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 12:50:27 +0200 Subject: [PATCH 058/281] added cache class --- .../service/cf/CalculatedFieldCache.java | 43 +++ .../cf/DefaultCalculatedFieldCache.java | 212 +++++++++++++ ...efaultCalculatedFieldExecutionService.java | 279 ++++++++++-------- .../thingsboard/server/common/msg/TbMsg.java | 15 +- common/proto/src/main/proto/queue.proto | 29 +- .../engine/telemetry/TbMsgAttributesNode.java | 1 + 6 files changed, 454 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..f953e57cc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; +import java.util.Set; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..1fc2a90e07 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,212 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + + @PostConstruct + public void init() { + // to discuss: fetch on start or fetch on demand + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + } + + @Override + public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedFieldFetchLock.lock(); + try { + calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (calculatedField != null) { + calculatedFields.put(calculatedFieldId, calculatedField); + log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); + return calculatedField; + } + + @Override + public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + List cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cfLinks != null) { + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); + return cfLinks; + } + + @Override + public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + List cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (cfLinks != null) { + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); + return cfLinks; + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + calculatedFieldFetchLock.lock(); + try { + entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> + throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e5c119bba6..ff78d2925a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,7 +18,11 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -37,8 +41,23 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -46,10 +65,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -67,8 +84,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,7 +94,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -90,10 +106,9 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -103,13 +118,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -123,20 +133,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy public void stop() { + super.stop(); if (calculatedFieldExecutor != null) { calculatedFieldExecutor.shutdownNow(); } @@ -183,7 +184,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas for (CalculatedField cf : partition) { EntityId cfEntityId = cf.getEntityId(); if (isProfileEntity(cfEntityId)) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { restoreState(cf, cfEntityId); @@ -206,7 +207,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); - calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", cf.getId()); } else { @@ -220,7 +220,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @@ -235,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -245,8 +244,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); - calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -258,7 +256,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); @@ -290,8 +288,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); + Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -316,8 +314,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -326,7 +324,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } @@ -353,28 +351,61 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entry.getKey(); } + private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { + try { + String type = objectProto.getType(); + String value = objectProto.getValue(); + return switch (type) { + case "java.lang.String" -> value; + case "java.lang.Integer" -> Integer.parseInt(value); + case "java.lang.Long" -> Long.parseLong(value); + case "java.lang.Double" -> Double.parseDouble(value); + case "java.lang.Boolean" -> Boolean.parseBoolean(value); + default -> throw new IllegalArgumentException("Unsupported object type: " + type); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); + return null; + } + } + @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - String state = proto.getState(); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + List calculatedFieldIds = new ArrayList<>(); + for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId cfId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(cfId); + } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); - if (calculatedFieldEntityCtx != null) { - states.put(ctxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + Map argumentsMap = new HashMap<>(); + proto.getArgumentsMap().forEach((key, entryProto) -> { + ArgumentEntry argumentEntry; + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { + Object value = deserializeObjectProto(objectProto); + tsRollingArgumentEntry.getTsRecords().put(ts, value); + }); + argumentEntry = tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); + Object value = deserializeObjectProto(singleRecordProto.getValue()); + argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); } else { - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } - } else { - log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); - } + argumentsMap.put(key, argumentEntry); + }); + + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -389,8 +420,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(oldProfileId).remove(entityId); - profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { @@ -400,7 +431,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); @@ -419,12 +450,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(profileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); List calculatedFieldIds = Stream.concat( - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); calculatedFieldIds.forEach(cfId -> { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); @@ -433,12 +462,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -446,7 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -455,20 +484,45 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .setEntityType(entityId.getEntityType().name()) .setEntityIdMSB(entityId.getId().getMostSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (calculatedFieldState != null) { - msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> { + TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { + TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() + .setType(value.getClass().getName()) + .setValue(value.toString()); + tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); + }); + + argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(TransportProtos.ObjectProto.newBuilder() + .setType(singleValueArgumentEntry.getValue().getClass().getName()) + .setValue(singleValueArgumentEntry.getValue().toString()) + .build()); + argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); + } + + msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); + }); } + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); - calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -483,8 +537,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFields.remove(calculatedFieldId); - calculatedFieldsCtx.remove(calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) @@ -497,28 +550,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); - } - - private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { - return switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -537,7 +568,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -562,7 +593,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -651,46 +682,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Predicate> allArgsPresent = (args) -> - args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } + }; - if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); - } - } - }; + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - return; + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; + } + performUpdateState.accept(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } - performUpdateState.accept(state); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { @@ -724,6 +756,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); + if (calculatedFieldIds == null) { + calculatedFieldIds = new ArrayList<>(); + } if (calculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 993fa24ff3..0805175f77 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -191,6 +192,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -495,8 +506,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 3df19e313e..f6f7afd5d8 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,7 +817,34 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - string state = 8; + repeated CalculatedFieldIdProto calculatedFields = 8; + map arguments = 9; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message ArgumentEntryProto { + oneof entry_type { + TsRollingProto tsRecords = 1; + SingleValueProto singleValue = 2; + } +} + +message TsRollingProto { + map tsRecords = 1; +} + +message SingleValueProto { + int64 ts = 1; + ObjectProto value = 2; +} + +message ObjectProto { + string type = 1; + string value = 2; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index e83a125265..20d0dda42f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,6 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(callback) .build()); } From 2dfbe2240d4bd22904e3d5edd6a7edeb15b0d1a5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 16:25:01 +0200 Subject: [PATCH 059/281] added implementation to handle profile events when not my partition --- ...efaultCalculatedFieldExecutionService.java | 510 +++++++++--------- .../server/common/util/ProtoUtils.java | 40 ++ common/proto/src/main/proto/queue.proto | 14 +- 3 files changed, 306 insertions(+), 258 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index ff78d2925a..6e7536825a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -84,7 +84,6 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +97,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; @TbCoreComponent @Service @@ -229,6 +230,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (!tpi.isMyPartition()) { + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); + log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); + } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -277,6 +284,54 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + } else { + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } + calculatedFieldCache.evict(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + } + @Override public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { @@ -288,8 +343,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); - Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> { + cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + }); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -312,6 +369,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -334,75 +408,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - - private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { - try { - String type = objectProto.getType(); - String value = objectProto.getValue(); - return switch (type) { - case "java.lang.String" -> value; - case "java.lang.Integer" -> Integer.parseInt(value); - case "java.lang.Long" -> Long.parseLong(value); - case "java.lang.Double" -> Double.parseDouble(value); - case "java.lang.Boolean" -> Boolean.parseBoolean(value); - default -> throw new IllegalArgumentException("Unsupported object type: " + type); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); - return null; - } - } - @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - List calculatedFieldIds = new ArrayList<>(); - for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { - CalculatedFieldId cfId = new CalculatedFieldId(new UUID( - cfIdProto.getCalculatedFieldIdMSB(), - cfIdProto.getCalculatedFieldIdLSB() - )); - calculatedFieldIds.add(cfId); + + if (proto.getClear()) { + clearState(tenantId, calculatedFieldId, entityId); + return; } - Map argumentsMap = new HashMap<>(); - proto.getArgumentsMap().forEach((key, entryProto) -> { - ArgumentEntry argumentEntry; - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { - Object value = deserializeObjectProto(objectProto); - tsRollingArgumentEntry.getTsRecords().put(ts, value); - }); - argumentEntry = tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); - Object value = deserializeObjectProto(singleRecordProto.getValue()); - argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - argumentsMap.put(key, argumentEntry); - }); + List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList(); + Map argumentsMap = proto.getArgumentsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); @@ -424,16 +446,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + .forEach(cfId -> clearState(tenantId, cfId, entityId)); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); } catch (Exception e) { @@ -455,16 +468,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); - calculatedFieldIds.forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); @@ -475,94 +479,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> { - TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { - TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() - .setType(value.getClass().getName()) - .setValue(value.toString()); - tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); - }); - - argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(TransportProtos.ObjectProto.newBuilder() - .setType(singleValueArgumentEntry.getValue().getClass().getName()) - .setValue(singleValueArgumentEntry.getValue().toString()) - .build()); - argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); - } - - msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); - }); - } - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - callback.onSuccess(); - shouldReinit = false; + sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } - return shouldReinit; - } - - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } - calculatedFieldCache.evict(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { @@ -605,82 +531,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( - Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor) - ); - case "TS_LATEST" -> transformSingleValueArgument( - Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { - long currentTime = System.currentTimeMillis(); - long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); - } - - private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createSingleValueArgument(kvEntry.get()); - } else { - return SingleValueArgumentEntry.EMPTY; - } - }, calculatedFieldCallbackExecutor); - } - - private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); - String defaultValue = argument.getDefaultValue(); - if (NumberUtils.isParsable(defaultValue)) { - return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); - } - if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { - return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); - } - return new StringDataEntry(key, defaultValue); - } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); @@ -742,14 +592,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); - } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); @@ -770,6 +612,166 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = isProfileEntity(argumentEntityId) + ? targetEntityId + : argumentEntityId; + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + } + + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); + } + if (calculatedFieldIds != null) { + calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build() + )); + } + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + + private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) + .setClear(true) + .build(); + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); + } + + private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( + TenantId tenantId, + CalculatedFieldId calculatedFieldId, + EntityId entityId + ) { + return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + } + + private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { + TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> + tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) + ); + argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + argumentProtoBuilder.setSingleValue( + TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(toObjectProto(singleValueArgumentEntry.getValue())) + .build() + ); + } + + return argumentProtoBuilder.build(); + } + + private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> + tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) + ); + return tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + } else { + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); + } + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 264b23f118..ec17914fd8 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1183,6 +1183,46 @@ public class ProtoUtils { return builder.build(); } + public static TransportProtos.ObjectProto toObjectProto(Object value) { + if (value == null) { + throw new IllegalArgumentException("Cannot convert null to ObjectProto"); + } + + TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); + + if (value instanceof String) { + builder.setStringValue((String) value); + } else if (value instanceof Integer) { + builder.setIntValue((Integer) value); + } else if (value instanceof Long) { + builder.setLongValue((Long) value); + } else if (value instanceof Double) { + builder.setDoubleValue((Double) value); + } else if (value instanceof Boolean) { + builder.setBoolValue((Boolean) value); + } else { + throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); + } + + return builder.build(); + } + + public static Object fromObjectProto(TransportProtos.ObjectProto proto) { + try { + return switch (proto.getValueCase()) { + case STRINGVALUE -> proto.getStringValue(); + case INTVALUE -> proto.getIntValue(); + case LONGVALUE -> proto.getLongValue(); + case DOUBLEVALUE -> proto.getDoubleValue(); + case BOOLVALUE -> proto.getBoolValue(); + case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", proto, e); + return null; + } + } + private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f6f7afd5d8..581f9eb9f2 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,8 +817,9 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - repeated CalculatedFieldIdProto calculatedFields = 8; - map arguments = 9; + bool clear = 8; + repeated CalculatedFieldIdProto calculatedFields = 9; + map arguments = 10; } message CalculatedFieldIdProto { @@ -843,8 +844,13 @@ message SingleValueProto { } message ObjectProto { - string type = 1; - string value = 2; + oneof value { + string stringValue = 1; + int32 intValue = 2; + int64 longValue = 3; + double doubleValue = 4; + bool boolValue = 5; + } } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From befd4cc9c69749dc510c5ff6ae2ff67713264d06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 10:56:20 +0200 Subject: [PATCH 060/281] added check for update values for ts rolling when rocksdb fails --- .../service/cf/DefaultCalculatedFieldCache.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 13 +++++++------ .../cf/ctx/state/BaseCalculatedFieldState.java | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 1fc2a90e07..dd2ab3857c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -100,11 +100,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cfLinks != null) { calculatedFieldLinks.put(calculatedFieldId, cfLinks); @@ -122,11 +122,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); if (cfLinks != null) { entityIdCalculatedFieldLinks.put(entityId, cfLinks); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6e7536825a..808c1ea8d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -556,20 +556,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + + if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); - return; + } else { + performUpdateState.accept(state); } - performUpdateState.accept(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index b73ac51798..462fa19b17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -24,6 +24,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; public BaseCalculatedFieldState() { + arguments = new HashMap<>(); } @Override From e787b805dae819f083876168e0a960bd5c801ae1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 17:05:27 +0200 Subject: [PATCH 061/281] added locking and logic to avoid iteration order issue in map --- ...efaultCalculatedFieldExecutionService.java | 68 +++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 5 +- .../ctx/state/BaseCalculatedFieldState.java | 49 ++++++------- .../cf/ctx/state/CalculatedFieldCtx.java | 8 ++- .../ctx/state/ScriptCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 ++-- .../cf/ctx/state/TsRollingArgumentEntry.java | 8 ++- common/proto/src/main/proto/queue.proto | 1 + 8 files changed, 87 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 808c1ea8d8..af4b3e95e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -119,6 +120,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; + private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -536,40 +539,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); + lock.lock(); + + try { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } - } - }; + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { + if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } + } finally { + lock.unlock(); } } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); @@ -747,7 +757,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return tsRollingArgumentEntry; } else if (entryProto.hasSingleValue()) { TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); } else { throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index ba7e094f77..dd56405352 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -import java.util.TreeMap; -import java.util.stream.Collectors; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -48,8 +46,7 @@ public interface ArgumentEntry { } static ArgumentEntry createTsRollingArgument(List kvEntries) { - return new TsRollingArgumentEntry(kvEntries.stream(). - collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); + return new TsRollingArgumentEntry(kvEntries); } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 462fa19b17..be7319b930 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -37,34 +36,30 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - AtomicBoolean stateUpdated = new AtomicBoolean(false); - argumentValues.forEach((key, argumentEntry) -> { - ArgumentEntry existingArgumentEntry = arguments.get(key); - if (existingArgumentEntry != null) { - if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); - } - } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); - stateUpdated.set(true); - } - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - stateUpdated.set(true); - } - } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { + if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { + existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); + } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { + existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry + && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + arguments.put(key, newEntry.copy()); + } else { + arguments.put(key, newEntry.copy()); } - } else { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); + stateUpdated = true; } - }); - return stateUpdated.get(); + } + + return stateUpdated; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 2cd5c68144..d54a3220ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -36,6 +38,7 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final List argKeys; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -48,10 +51,11 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.argKeys = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } @@ -65,7 +69,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - arguments.keySet().toArray(new String[0]) + argKeys.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 87429050de..de7c514786 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + Object[] args = ctx.getArgKeys().stream() + .map(key -> arguments.get(key).getValue()) + .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3f4fd5bdce..20b531f562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,11 +32,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; + private long version; + public SingleValueArgumentEntry(KvEntry entry) { - if (entry instanceof TsKvEntry) { - this.ts = ((TsKvEntry) entry).getTs(); - } else if (entry instanceof AttributeKvEntry) { - this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); } this.value = entry.getValue(); } @@ -66,7 +70,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value); + return new SingleValueArgumentEntry(this.ts, this.value, this.version); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 4ffa391550..64de2f8c8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,16 +16,20 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.util.List; import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor +@AllArgsConstructor @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { @@ -35,8 +39,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); - public TsRollingArgumentEntry(TreeMap tsRecords) { - addAllTsRecords(tsRecords); + public TsRollingArgumentEntry(List kvEntries) { + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); } /** diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 581f9eb9f2..1f66621a5b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,6 +841,7 @@ message TsRollingProto { message SingleValueProto { int64 ts = 1; ObjectProto value = 2; + int64 version = 3; } message ObjectProto { From 5f088a751e2fc18435fecfeee603c592ef412714 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 27 Dec 2024 18:20:20 +0200 Subject: [PATCH 062/281] refactored code --- .../cf/CalculatedFieldExecutionService.java | 9 +- .../service/cf/CalculatedFieldResult.java | 5 +- ...efaultCalculatedFieldExecutionService.java | 122 ++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- ...gumentType.java => ArgumentEntryType.java} | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 8 +- .../ctx/state/SingleValueArgumentEntry.java | 6 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 +- ...CalculatedFieldAttributeUpdateRequest.java | 38 ++++++ ...CalculatedFieldTelemetryUpdateRequest.java | 38 ++++++ ...alculatedFieldTimeSeriesUpdateRequest.java | 42 ++++++ .../DefaultTelemetrySubscriptionService.java | 6 +- .../CalculatedFieldControllerTest.java | 6 +- .../cf/CalculatedFieldLinkConfiguration.java | 4 +- .../data/cf/configuration/Argument.java | 2 +- .../data/cf/configuration/ArgumentType.java | 22 ++++ .../BaseCalculatedFieldConfiguration.java | 45 ++++--- .../common/data/cf/configuration/Output.java | 2 +- .../data/cf/configuration/OutputType.java | 22 ++++ .../server/dao/service/AssetServiceTest.java | 6 +- .../service/CalculatedFieldServiceTest.java | 6 +- .../dao/service/CustomerServiceTest.java | 6 +- .../server/dao/service/DeviceServiceTest.java | 6 +- 23 files changed, 302 insertions(+), 107 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{ArgumentType.java => ArgumentEntryType.java} (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index ee9b6d115c..e4b0a7ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); + void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index e8ea318bf6..52e0a0151b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -17,17 +17,18 @@ package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import java.util.Map; @Data public final class CalculatedFieldResult { - private final String type; + private final OutputType type; private final AttributeScope scope; private final Map resultMap; - public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + public CalculatedFieldResult(OutputType type, AttributeScope scope, Map resultMap) { this.type = type; this.scope = scope; this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index af4b3e95e5..27db9ae290 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,12 +35,15 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,7 +51,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -79,11 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +96,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -120,12 +123,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -336,28 +341,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> { - cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); - }); + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); + List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); + List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + EntityId profileId = getProfileId(tenantId, entityId); + + List cfLinks = Stream.concat( + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), + profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() + ).toList(); + cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); + Map telemetryKeys = getTelemetryKeysFromLink(link, scope); Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), + entry -> getMappedKey(entry, telemetryKeys), entry -> entry, (v1, v2) -> v1 )); @@ -368,25 +374,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } } catch (Exception e) { - log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + log.trace("Failed to update telemetry.", e); } } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); + private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { + return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } + + private String getMappedKey(KvEntry entry, Map telemetry) { + return telemetry.entrySet().stream() + .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { @@ -539,17 +544,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); - lock.lock(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - try { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && @@ -564,12 +566,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) @@ -578,9 +580,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } - } finally { - lock.unlock(); - } + return calculatedFieldEntityCtx; + }); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } @@ -605,9 +606,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { - String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); if (calculatedFieldIds == null) { calculatedFieldIds = new ArrayList<>(); @@ -649,19 +650,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); - case "TS_LATEST" -> transformSingleValueArgument( + case TS_LATEST -> transformSingleValueArgument( Futures.transform( timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } @@ -801,4 +801,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); } + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index dd56405352..b261840bfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -35,7 +35,7 @@ import java.util.List; public interface ArgumentEntry { @JsonIgnore - ArgumentType getType(); + ArgumentEntryType getType(); Object getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 360529a7e9..1a0dfb5ac7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -public enum ArgumentType { +public enum ArgumentEntryType { SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index be7319b930..bc9a421e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -49,8 +49,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry - && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { +// Long existingVersion = existingSingleValueEntry.getVersion(); +// Long newVersion = singleValueEntry.getVersion(); +// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { +// arguments.put(key, newEntry.copy()); +// } arguments.put(key, newEntry.copy()); } else { arguments.put(key, newEntry.copy()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 20b531f562..8d5080e90f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,7 +32,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private long version; + private Long version; public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { @@ -54,8 +54,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.SINGLE_VALUE; + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 64de2f8c8a..1118e3af13 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -51,8 +51,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.TS_ROLLING; + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java new file mode 100644 index 0000000000..8479ff37d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private AttributeScope scope; + private List kvEntries; + private List calculatedFieldIds; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java new file mode 100644 index 0000000000..3c28833f31 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.List; + +public interface CalculatedFieldTelemetryUpdateRequest { + + TenantId getTenantId(); + + EntityId getEntityId(); + + AttributeScope getScope(); + + List getKvEntries(); + + List getCalculatedFieldIds(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java new file mode 100644 index 0000000000..987d899465 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private List kvEntries; + private List calculatedFieldIds; + + @Override + public AttributeScope getScope() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index e35d3cead6..bf3076be5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,6 +50,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -152,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); return saveFuture; } @@ -168,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ba1dfb1fec..5d1467974d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -24,8 +24,10 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -140,7 +142,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +151,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index c5f81cd572..78513c8b74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -23,7 +23,9 @@ import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private Map attributes = new HashMap<>(); + private Map clientAttributes = new HashMap<>(); + private Map serverAttributes = new HashMap<>(); + private Map sharedAttributes = new HashMap<>(); private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..4dac866219 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -24,7 +24,7 @@ public class Argument { private EntityId entityId; private String key; - private String type; + private ArgumentType type; private AttributeScope scope; private String defaultValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..17e2315b52 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index ac36991a61..8c86b6c552 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -65,19 +65,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - for (Map.Entry entry : arguments.entrySet()) { - Argument argument = entry.getValue(); - if (argument.getEntityId().equals(entityId)) { - switch (argument.getType()) { - case "ATTRIBUTE": - linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); - break; - case "TS_LATEST", "TS_ROLLING": - linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); - break; - } - } - } + arguments.entrySet().stream() + .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .forEach(entry -> { + Argument argument = entry.getValue(); + String argumentKey = entry.getKey(); + + switch (argument.getType()) { + case ATTRIBUTE -> { + switch (argument.getScope()) { + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + } + } + case TS_LATEST, TS_ROLLING -> + linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + } + }); return linkConfiguration; } @@ -98,7 +103,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityId", entityId.toString()); } argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); + argumentNode.put("type", String.valueOf(argument.getType())); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); @@ -112,7 +117,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); - outputNode.put("type", output.getType()); + outputNode.put("type", String.valueOf(output.getType())); if (output.getScope() != null) { outputNode.put("scope", String.valueOf(output.getScope())); } @@ -141,7 +146,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); + JsonNode type = argumentNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + argument.setType(ArgumentType.valueOf(type.asText())); + } JsonNode scope = argumentNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); @@ -169,7 +177,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); - output.setType(outputNode.get("type").asText()); + JsonNode type = outputNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + output.setType(OutputType.valueOf(type.asText())); + } JsonNode scope = outputNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { output.setScope(AttributeScope.valueOf(scope.asText())); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 46257d1ccc..12cf97338a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.AttributeScope; public class Output { private String name; - private String type; + private OutputType type; private AttributeScope scope; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..c248bc8042 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index b0870f3dc5..9a0b9222f0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -33,7 +33,9 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -884,7 +886,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +895,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 9a1719e715..5a8f7a2383 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -26,8 +26,10 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -153,7 +155,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +164,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 6671e0e821..d0ee833261 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -34,7 +34,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -379,7 +381,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +390,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2f8686bc3..5b060ae145 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1222,7 +1224,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1233,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); From 8f87caba148aac97973b0d0f212d998957eb5a00 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 30 Dec 2024 16:50:01 +0200 Subject: [PATCH 063/281] implemented logic to handle profile updates even if no cf exists by them --- ...efaultCalculatedFieldExecutionService.java | 11 +++--- .../queue/DefaultTbClusterService.java | 36 ++++--------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 27db9ae290..96cbdfc73b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -542,6 +542,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + Map argumentsMap = new HashMap<>(argumentValues); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -550,7 +551,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { + if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); @@ -564,17 +565,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); } else { @@ -583,7 +584,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index d57a5266bf..6316d1fdb2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -150,7 +149,6 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; - private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -393,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); + sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -402,17 +400,10 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } - private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, false, true); - } - } - @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -633,13 +624,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -653,29 +644,14 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); - boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); - if (cfExistsByOldProfile || cfExistsByNewProfile) { - sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); - } - } - - private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, true, false); - } - } - @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { From 63b79b7242d1c695cbacb46d156495247aa38ba8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 31 Dec 2024 16:15:42 +0200 Subject: [PATCH 064/281] added completable future to prevent reading old value from while updating is in progress --- .../DefaultCalculatedFieldExecutionService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 96cbdfc73b..976139e8bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -94,6 +94,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; @@ -543,13 +544,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + CompletableFuture updateFuture = new CompletableFuture<>(); + Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); @@ -561,6 +566,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } + updateFuture.complete(null); }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -570,7 +576,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -581,6 +586,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } + + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } + return calculatedFieldEntityCtx; }); } else { From 4ebb68ded68159fcda166e06499bbd75e6bbe53e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 3 Jan 2025 16:00:35 +0200 Subject: [PATCH 065/281] handled profile updates in cluster --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 25 +++++++++++ ...efaultCalculatedFieldExecutionService.java | 44 +++++++++--------- .../queue/DefaultTbClusterService.java | 26 ++++++----- .../queue/DefaultTbCoreConsumerService.java | 38 ++++++++++++++-- .../queue/DefaultTbEdgeConsumerService.java | 8 ++-- .../DefaultTbRuleEngineConsumerService.java | 6 ++- .../processing/AbstractConsumerService.java | 9 ++++ common/proto/src/main/proto/queue.proto | 2 + docker/docker-compose.cluster.yml | 45 +++++++++++++++++++ 10 files changed, 166 insertions(+), 39 deletions(-) create mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index f953e57cc3..ad683c324c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -34,6 +34,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index dd2ab3857c..a4077b599b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -141,6 +141,31 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } + + @Override + public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); + calculatedFieldFetchLock.lock(); + try { + List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + if (cfLinks != null && !cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { + if (existingList == null) { + existingList = new ArrayList<>(); + } else if (!(existingList instanceof ArrayList)) { + existingList = new ArrayList<>(existingList); + } + existingList.add(link); + return existingList; + }); + }); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 976139e8bd..2f55de6b77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,7 +71,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -105,7 +104,6 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; -@TbCoreComponent @Service @Slf4j @RequiredArgsConstructor @@ -117,7 +115,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + // private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -213,8 +211,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedField cf, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); +// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); @@ -313,13 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFieldCache.evict(calculatedFieldId); +// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); - rocksDBService.deleteAll(statesToRemove); +// rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -414,7 +413,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @Override @@ -423,7 +421,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - + log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); if (proto.getClear()) { clearState(tenantId, calculatedFieldId, entityId); return; @@ -431,7 +429,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List calculatedFieldIds = proto.getCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList(); + .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); @@ -451,8 +449,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -472,15 +470,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); + + List calculatedFieldIds = Stream.concat( calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -494,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); +// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -558,13 +559,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); +// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } updateFuture.complete(null); }; @@ -633,6 +635,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldIds.add(calculatedFieldId); TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); } @@ -715,6 +718,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); } + log.info("Sending calculated field state msg from entityId [{}]", entityId); clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } @@ -790,11 +794,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); +// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); +// if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); +// } +// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 6316d1fdb2..551b0ddbc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -391,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -400,7 +400,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); + handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -563,7 +563,9 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) - || entityType.equals(EntityType.NOTIFICATION_RULE)) { + || entityType.equals(EntityType.NOTIFICATION_RULE) + || entityType.equals(EntityType.CALCULATED_FIELD) + ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { @@ -624,13 +626,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -644,10 +646,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -792,8 +794,8 @@ public class DefaultTbClusterService implements TbClusterService { public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { @@ -809,7 +811,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -822,10 +824,12 @@ public class DefaultTbClusterService implements TbClusterService { builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -838,6 +842,8 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setDeleted(deleted); TransportProtos.ProfileEntityMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 1be25b308f..6baa75b3ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -87,6 +88,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; @@ -109,6 +111,7 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -181,8 +184,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); + if (added) { + entitiesByProfile.add(entityId); + } else { + entitiesByProfile.remove(entityId); + } + } + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); @@ -688,12 +718,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index f219d7ae69..35c7894f2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -91,7 +91,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService future = edgeCtx.getTenantProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); case NOTIFICATION_RULE, NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE -> future = edgeCtx.getNotificationEdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case TB_RESOURCE -> future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DOMAIN, OAUTH2_CLIENT -> future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case TB_RESOURCE -> + future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case DOMAIN, OAUTH2_CLIENT -> + future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); default -> { future = Futures.immediateFuture(null); log.warn("[{}] Edge event type [{}] is not designed to be pushed to edge", tenantId, type); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 73810949b3..7d4d975cb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; @@ -83,8 +84,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - JwtSettingsService jwtSettingsService) { - super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; this.queueService = queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 95fc8cb843..2aaed13ec1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -43,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.TbPackCallback; @@ -68,6 +70,7 @@ public abstract class AbstractConsumerService Date: Mon, 6 Jan 2025 14:15:32 +0200 Subject: [PATCH 066/281] changed partititoning implementation --- ...efaultCalculatedFieldExecutionService.java | 180 ++++++++---------- ...CalculatedFieldAttributeUpdateRequest.java | 13 +- ...CalculatedFieldTelemetryUpdateRequest.java | 9 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 9 +- .../DefaultTelemetrySubscriptionService.java | 20 +- .../thingsboard/server/common/msg/TbMsg.java | 32 ++-- common/proto/src/main/proto/queue.proto | 2 +- docker/docker-compose.cluster.yml | 45 ----- .../engine/api/AttributesSaveRequest.java | 10 +- .../engine/api/TimeseriesSaveRequest.java | 10 +- .../engine/telemetry/TbMsgAttributesNode.java | 2 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- 12 files changed, 145 insertions(+), 189 deletions(-) delete mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2f55de6b77..f7bcb4e678 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,6 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -98,7 +97,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; @@ -115,7 +113,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - // private final RocksDBService rocksDBService; + private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -168,35 +166,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiCalculatedFieldMap = new HashMap<>(); + Map> tpiTargetEntityMap = new HashMap<>(); for (CalculatedField cf : cfs) { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", - cf.getId(), cf.getTenantId(), e.getMessage()); - continue; - } - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + + Consumer resolvePartition = entityId -> { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + } + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", + entityId, cf.getTenantId(), e.getMessage()); + } + }; + + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); + } else { + resolvePartition.accept(cfEntityId); } } - for (var entry : tpiCalculatedFieldMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { + for (var entry : tpiTargetEntityMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); var future = calculatedFieldExecutor.submit(() -> { try { - for (CalculatedField cf : partition) { - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) - .forEach(entityId -> restoreState(cf, entityId)); - } else { - restoreState(cf, cfEntityId); - } + for (CalculatedFieldEntityCtxId ctxId : partition) { + restoreState(ctxId.cfId(), ctxId.entityId()); } } catch (Throwable t) { log.error("Unexpected exception while restoring CalculatedField states", t); @@ -209,17 +210,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(CalculatedField cf, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); -// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + private void restoreState(UUID calculatedFieldId, UUID entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); - String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", cf.getId()); + log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); } } @@ -238,12 +238,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (!tpi.isMyPartition()) { - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); - log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); - } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -307,18 +301,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } -// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); -// rocksDBService.deleteAll(statesToRemove); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -345,22 +333,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); - AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); - List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); - List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); - List cfLinks = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), - profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() - ).toList(); - - cfLinks.forEach(link -> { + getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = getTelemetryKeysFromLink(link, scope); - Map updatedTelemetry = telemetry.stream() + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), @@ -369,7 +349,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } }); } @@ -378,14 +359,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { - return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); - }; - } - private String getMappedKey(KvEntry entry, Map telemetry) { return telemetry.entrySet().stream() .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) @@ -394,7 +367,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .orElse(entry.getKey()); } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); @@ -406,12 +379,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); + default -> + updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); } } @@ -427,14 +401,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } - List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -449,9 +423,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -470,18 +441,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); - - - List calculatedFieldIds = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) - ).toList(); - calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); + getCalculatedFieldLinks(tenantId, entityId, profileId) + .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -490,12 +454,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); -// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -541,12 +505,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -559,12 +523,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); -// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } @@ -599,17 +563,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); } } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); } } @@ -620,20 +584,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (calculatedFieldIds == null) { - calculatedFieldIds = new ArrayList<>(); - } - if (calculatedFieldIds.contains(calculatedFieldId)) { + if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } + List calculatedFieldIds = previousCalculatedFieldIds != null + ? new ArrayList<>(previousCalculatedFieldIds) + : new ArrayList<>(); calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { @@ -641,6 +605,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + if (profileId != null) { + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + } + return links; + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -704,13 +676,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); if (argumentValues != null) { argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); } - if (calculatedFieldIds != null) { - calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + if (previousCalculatedFieldIds != null) { + previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) @@ -794,11 +766,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { -// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); -// if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); -// } -// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 8479ff37d7..c56217b2ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -33,6 +35,15 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private EntityId entityId; private AttributeScope scope; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; + + @Override + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 3c28833f31..98062a08db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; +import java.util.Map; public interface CalculatedFieldTelemetryUpdateRequest { @@ -29,10 +30,10 @@ public interface CalculatedFieldTelemetryUpdateRequest { EntityId getEntityId(); - AttributeScope getScope(); - List getKvEntries(); - List getCalculatedFieldIds(); + List getPreviousCalculatedFieldIds(); + + Map getTelemetryKeysFromLink(CalculatedFieldLink link); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 987d899465..bd2161dca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -32,11 +33,11 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private TenantId tenantId; private EntityId entityId; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; @Override - public AttributeScope getScope() { - return null; + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return link.getConfiguration().getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf3076be5c..99cbd89494 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); return saveFuture; } @@ -170,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); } @Override @@ -243,7 +243,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) {} + public void onSuccess(@Nullable Void tmp) { + } @Override public void onFailure(Throwable t) { @@ -342,4 +343,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } + protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable T result) { + callback.accept(result); + } + + @Override + public void onFailure(Throwable t) { + } + }, tsCallBackExecutor); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 0805175f77..9d99f38c60 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -35,10 +35,10 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; /** * Created by ashvayka on 13.01.18. @@ -67,7 +67,7 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; @Getter(value = AccessLevel.NONE) @JsonIgnore @@ -117,7 +117,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List previousCalculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -144,7 +144,9 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; - this.calculatedFieldIds = calculatedFieldIds; + this.previousCalculatedFieldIds = previousCalculatedFieldIds != null + ? new CopyOnWriteArrayList<>(previousCalculatedFieldIds) + : new CopyOnWriteArrayList<>(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -192,8 +194,8 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } - if (msg.getCalculatedFieldIds() != null) { - for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + if (msg.getPreviousCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getPreviousCalculatedFieldIds()) { MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) @@ -216,7 +218,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; - List calculatedFieldIds = new ArrayList<>(); + List calculatedFieldIds = new CopyOnWriteArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -274,6 +276,7 @@ public final class TbMsg implements Serializable { /** * Checks if the message is still valid for processing. May be invalid if the message pack is timed-out or canceled. + * * @return 'true' if message is valid for processing, 'false' otherwise. */ public boolean isValid() { @@ -368,7 +371,7 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; - protected List calculatedFieldIds; + protected List previousCalculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; @@ -390,7 +393,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; - this.calculatedFieldIds = tbMsg.calculatedFieldIds; + this.previousCalculatedFieldIds = tbMsg.previousCalculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -413,8 +416,7 @@ public final class TbMsg implements Serializable { /** *

Deprecated: This should only be used when you need to specify a custom message type that doesn't exist in the {@link TbMsgType} enum. * Prefer using {@link #type(TbMsgType)} instead. - * - * */ + */ @Deprecated public TbMsgBuilder type(String type) { this.type = type; @@ -482,8 +484,8 @@ public final class TbMsg implements Serializable { return this; } - public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -498,7 +500,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, previousCalculatedFieldIds, ctx, callback); } public String toString() { @@ -506,7 +508,7 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1c4927c422..9cba62eecc 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -818,7 +818,7 @@ message CalculatedFieldStateMsgProto { int64 entityIdMSB = 6; int64 entityIdLSB = 7; bool clear = 8; - repeated CalculatedFieldIdProto calculatedFields = 9; + repeated CalculatedFieldIdProto previousCalculatedFields = 9; map arguments = 10; } diff --git a/docker/docker-compose.cluster.yml b/docker/docker-compose.cluster.yml deleted file mode 100644 index 4c74238a10..0000000000 --- a/docker/docker-compose.cluster.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright © 2016-2024 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3.0' - -services: - kafka: - restart: always - image: "bitnami/kafka:3.7.0" - ports: - - "9092:9092" - env_file: - - kafka.env - depends_on: - - zookeeper - zookeeper: - restart: always - image: "zookeeper:3.8.0" - ports: - - "2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - ZOO_ADMINSERVER_ENABLED: "false" - redis: - restart: always - image: bitnami/redis:7.2 - environment: - # ALLOW_EMPTY_PASSWORD is recommended only for development. - ALLOW_EMPTY_PASSWORD: "yes" - ports: - - '6379:6379' diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 9747a6033e..406b2d0d5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -41,7 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -55,7 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; private FutureCallback callback; Builder() {} @@ -103,8 +103,8 @@ public class AttributesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -128,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 12afa2d939..95eb788e5f 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -41,7 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -58,7 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; Builder() {} @@ -106,8 +106,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -131,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 20d0dda42f..ae2fce6575 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,7 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 386e56320a..89f7844313 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,7 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From 7d8a76ce7f8e1ca15e08709bb2fcba8446578fbf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Jan 2025 15:32:23 +0200 Subject: [PATCH 067/281] Draft of the review --- .../cf/DefaultCalculatedFieldCache.java | 10 ++++------ .../AbstractSubscriptionService.java | 9 +++++++-- .../DefaultTelemetrySubscriptionService.java | 20 +++++-------------- common/proto/src/main/proto/queue.proto | 7 +++++++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index a4077b599b..c4293b9edc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -122,16 +122,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - if (cfLinks != null) { - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index e26ede2bae..0bb8f76398 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -99,7 +100,11 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList } protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + addCallback(saveFuture, callback, wsCallBackExecutor); + } + + protected void addCallback(ListenableFuture saveFuture, Consumer callback, Executor executor) { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable T result) { callback.accept(result); @@ -108,7 +113,7 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList @Override public void onFailure(Throwable t) { } - }, wsCallBackExecutor); + }, executor); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 99cbd89494..f873e48774 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); + // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. + //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return saveFuture; } @@ -170,7 +172,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); + //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); } @Override @@ -343,17 +346,4 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } - protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable T result) { - callback.accept(result); - } - - @Override - public void onFailure(Throwable t) { - } - }, tsCallBackExecutor); - } - } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9cba62eecc..56f347f311 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -809,6 +809,13 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message ToServerB { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + repeated CfIdEntityIdPair links = 3; + value = 4; +} + message CalculatedFieldStateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; From e2ac2708b658c3802c9f15f9825bb621a3730f85 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 6 Jan 2025 17:37:33 +0200 Subject: [PATCH 068/281] changed CF links config --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 37 ++++++++++++++----- ...efaultCalculatedFieldExecutionService.java | 28 +++++++++++++- ...CalculatedFieldAttributeUpdateRequest.java | 21 +++++++---- ...CalculatedFieldTelemetryUpdateRequest.java | 4 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 16 +++++--- .../DefaultTelemetrySubscriptionService.java | 7 +--- .../server/dao/cf/CalculatedFieldService.java | 2 + .../BaseCalculatedFieldConfiguration.java | 17 +++++---- .../dao/cf/BaseCalculatedFieldService.java | 7 ++++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 ++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 13 files changed, 114 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ad683c324c..aa25565a34 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -30,6 +30,8 @@ public interface CalculatedFieldCache { CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c4293b9edc..40d569b040 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -56,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final DeviceService deviceService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); @@ -65,10 +66,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Getter private int initFetchPackSize; - @PostConstruct public void init() { - // to discuss: fetch on start or fetch on demand PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); @@ -97,19 +96,37 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return calculatedField; } + @Override + public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + List cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + calculatedFieldFetchLock.lock(); + try { + cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); + entityIdCalculatedFields.put(entityId, cfs); + log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); + return cfs; + } + @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cfLinks != null) { - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); @@ -139,7 +156,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } - @Override public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); @@ -225,6 +241,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f7bcb4e678..c0f7acf0a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -139,6 +140,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(() -> rocksDBService.getAll() + .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); } @PreDestroy @@ -337,11 +340,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); + // process by profile + if (profileId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, telemetryKeys), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); + } + }); + } + + // process by links getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), entry -> entry, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index c56217b2ce..25d2f57bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -37,12 +36,20 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.scope = request.getScope(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { return switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 98062a08db..29ee899ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLink link); + Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index bd2161dca1..6225286631 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -35,9 +34,16 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { - return link.getConfiguration().getTimeSeries(); + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { + return linkConfiguration.getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..8773564e5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,9 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)), tsCallBackExecutor); return saveFuture; } @@ -172,8 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 1e64fdac60..3a508a5c08 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -38,6 +38,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8c86b6c552..c3c4e32507 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -68,19 +68,22 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel arguments.entrySet().stream() .filter(entry -> entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument argument = entry.getValue(); + Argument tergetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (argument.getType()) { + switch (tergetArgument.getType()) { case ATTRIBUTE -> { - switch (argument.getScope()) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + switch (tergetArgument.getScope()) { + case CLIENT_SCOPE -> + linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + case SERVER_SCOPE -> + linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + case SHARED_SCOPE -> + linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); } }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0849414a0e..e650aec35e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -98,6 +98,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); } + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 5b3bcc2750..39663d0afc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -31,6 +31,8 @@ public interface CalculatedFieldDao extends Dao { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 9aa0aee428..816fa1546c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -28,6 +29,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index e3762f6157..20081299e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -55,6 +55,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 03c3341265724341aea3668f6f376a7e90d842f3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 12:39:49 +0200 Subject: [PATCH 069/281] added logic to send msgs to RE when not my partition --- .../server/controller/BaseController.java | 7 - .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 252 ++++++++++++++---- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 +- ...CalculatedFieldAttributeUpdateRequest.java | 6 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 6 +- .../TbRuleEngineQueueConsumerManager.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 14 +- .../server/common/util/ProtoUtils.java | 133 +++++++++ common/proto/src/main/proto/queue.proto | 29 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + 11 files changed, 386 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 4987096d17..139c61d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -366,9 +366,6 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; - @Autowired - protected CalculatedFieldService calculatedFieldService; - @Autowired protected NotificationTargetService notificationTargetService; @@ -998,10 +995,6 @@ public abstract class BaseController { return null; } - protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { - return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); - } - protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e4b0a7ca1e..6d1d459b9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -25,6 +25,8 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0f7acf0a3..0de3136439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,9 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; @@ -51,6 +53,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -67,6 +70,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -80,7 +84,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -102,6 +108,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -177,8 +184,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi; try { tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } } catch (Exception e) { log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", @@ -213,7 +220,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(UUID calculatedFieldId, UUID entityId) { + private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); @@ -232,7 +239,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } @Override @@ -243,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); + onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -293,7 +300,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { callback.onSuccess(); shouldReinit = false; @@ -301,12 +308,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -334,61 +341,147 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - EntityId profileId = getProfileId(tenantId, entityId); - - // process by profile - if (profileId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); - } + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + Map> tpiStatesToUpdate = new HashMap<>(); + + updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); + updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } + } - // process by links - getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + updateTelemetryForEntity(request, request.getEntityId(), tpiStates); + } + + private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { + updateTelemetryForEntity(request, profileId, tpiStates); + } + + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + if (targetEntity != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + }); + } + } else { + List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + }); + } + } + + private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + + private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + }); + } else { + updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); } }); - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); + } + + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, + EntityId entityId, + CalculatedFieldId calculatedFieldId, + CalculatedFieldTelemetryUpdateRequest request, + CalculatedFieldLinkConfiguration linkConfiguration) { + Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } } - private String getMappedKey(KvEntry entry, Map telemetry) { - return telemetry.entrySet().stream() - .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); + private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { + return kvEntries.stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), + entry -> entry, + (v1, v2) -> v1 + )); + } + + @Override + public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + + proto.getLinksList().forEach(ctxIdProto -> { + EntityId entityId = EntityIdFactory.getByTypeAndUuid( + ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + CalculatedFieldTelemetryUpdateRequest request = attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + + onTelemetryUpdate(request); + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { @@ -481,7 +574,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { @@ -537,7 +630,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); @@ -777,6 +870,57 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( + CalculatedFieldTelemetryUpdateRequest request, List links + ) { + TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); + + builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + + for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { + builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); + } + + if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { + builder.setScope(attributeUpdateRequest.getScope().name()); + } + + for (KvEntry entry : request.getKvEntries()) { + TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); + if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { + telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); + } + if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { + telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); + } + builder.addUpdatedTelemetry(telemetryBuilder.build()); + } + + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index f7c451efee..5fb90a3e46 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.cf.ctx; -import java.util.UUID; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; -public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 25d2f57bd6..a83cc0fc25 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; @@ -22,18 +23,19 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; private AttributeScope scope; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 6225286631..507daf386e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,23 +15,25 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..243a3adbf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -34,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -63,14 +64,18 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument tergetArgument = entry.getValue(); + Argument targetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (tergetArgument.getType()) { + switch (targetArgument.getType()) { case ATTRIBUTE -> { - switch (tergetArgument.getScope()) { + switch (targetArgument.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); } }); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index ec17914fd8..d332bac64f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -58,12 +59,14 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -627,6 +630,136 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static KvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { + if (telemetryProto.hasAttrKv()) { + return fromProto(telemetryProto.getAttrKv().getValue()); + } else if (telemetryProto.hasTsKv()) { + return fromProto(telemetryProto.getTsKv()); + } else { + throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); + } + } + + public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { + TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); + builder.setAttributeKey(key); + switch (scope) { + case CLIENT_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); + break; + case SERVER_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); + break; + case SHARED_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); + break; + default: + throw new IllegalArgumentException("Unsupported attribute scope: " + scope); + } + return builder.build(); + } + + public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { + return TransportProtos.AttributeKvProto.newBuilder() + .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) + .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) + .build(); + } + + public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { + TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); + builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setHasV(true) + .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setHasV(true) + .setLongV(attributeKvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setHasV(true) + .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setHasV(true) + .setStringV(attributeKvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setHasV(true) + .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); + break; + default: + builder.setHasV(false); + throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); + } + if (attributeKvEntry.getKey() != null) { + builder.setKey(attributeKvEntry.getKey()); + } + if (attributeKvEntry.getVersion() != null) { + builder.setVersion(attributeKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + return TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)) + .setVersion(tsKvEntry.getVersion()) + .build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 56f347f311..685ca47719 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,6 +183,18 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } +message AttributeKvProto { + AttributeKey key = 1; + AttributeValueProto value = 2; +} + +message TelemetryProto { + oneof proto { + AttributeKvProto attrKv = 1; + TsKvProto tsKv = 2; + } +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -809,11 +821,21 @@ message ProfileEntityMsgProto { bool deleted = 10; } -message ToServerB { +message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CfIdEntityIdPair links = 3; - value = 4; + repeated CalculatedFieldEntityCtxIdProto links = 3; + repeated CalculatedFieldIdProto previousCalculatedFields = 4; + string scope = 5; + repeated TelemetryProto updatedTelemetry = 6; +} + +message CalculatedFieldEntityCtxIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; } message CalculatedFieldStateMsgProto { @@ -1655,6 +1677,7 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; + TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; } message ToRuleEngineNotificationMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index e650aec35e..36bc3d038a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -247,6 +247,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); return cfConfig.getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) .map(referencedEntityId -> { CalculatedFieldLink link = new CalculatedFieldLink(); link.setTenantId(tenantId); From 46180e33d70761ea18f93238420e8ed1cf97f31e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 17:20:20 +0200 Subject: [PATCH 070/281] cache refactoring --- .../service/cf/CalculatedFieldCache.java | 16 +- .../cf/DefaultCalculatedFieldCache.java | 132 +++++++-------- ...efaultCalculatedFieldExecutionService.java | 156 ++++++++++-------- .../processing/AbstractConsumerService.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 21 +++ .../CalculatedFieldConfiguration.java | 7 + common/proto/src/main/proto/queue.proto | 11 +- .../dao/cf/BaseCalculatedFieldService.java | 18 +- .../dao/sql/cf/CalculatedFieldRepository.java | 3 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 2 +- 10 files changed, 195 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index aa25565a34..7394c95f08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -28,20 +28,24 @@ import java.util.Set; public interface CalculatedFieldCache { - CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void evict(CalculatedFieldId calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 40d569b040..71fc3eff67 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -70,98 +71,44 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + calculatedFields.values().forEach(cf -> + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + ); } @Override - public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedFieldFetchLock.lock(); - try { - calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (calculatedField != null) { - calculatedFields.put(calculatedFieldId, calculatedField); - log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); - return calculatedField; + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); } @Override - public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - List cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - calculatedFieldFetchLock.lock(); - try { - cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); - entityIdCalculatedFields.put(entityId, cfs); - log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); - return cfs; + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { - List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); } @Override - public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); calculatedFieldFetchLock.lock(); try { - List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + List cfLinks = getCalculatedFieldLinks(calculatedFieldId); if (cfLinks != null && !cfLinks.isEmpty()) { cfLinks.forEach(link -> { entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { @@ -181,14 +128,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, ctx); @@ -236,6 +183,42 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void evict(CalculatedFieldId calculatedFieldId) { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); @@ -243,7 +226,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFieldLinks.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); - entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0de3136439..dd11c799c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -253,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -263,7 +264,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -297,7 +298,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -345,17 +346,27 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); - Map> tpiStatesToUpdate = new HashMap<>(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); - updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); - updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (tpi.isMyPartition()) { - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - }); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + }); + } + } else { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -363,30 +374,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - updateTelemetryForEntity(request, request.getEntityId(), tpiStates); - } - - private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { - updateTelemetryForEntity(request, profileId, tpiStates); - } - - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - if (targetEntity != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); - }); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + if (cfTargetEntityId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); }); } } @@ -405,14 +400,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { @@ -451,33 +446,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(request); + return; + } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = EntityIdFactory.getByTypeAndUuid( - ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - CalculatedFieldTelemetryUpdateRequest request = attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - onTelemetryUpdate(request); + CalculatedFieldLinkConfiguration linkConfiguration + = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); + + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); @@ -486,8 +470,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -524,7 +508,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); @@ -559,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(tenantId, entityId, profileId) + getCalculatedFieldLinks(entityId, profileId) .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -585,7 +569,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -722,10 +706,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); } return links; } @@ -870,13 +854,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { + return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); + } + + ; + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(request.getEntityId().getEntityType().name()) + .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -904,6 +897,31 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return builder.build(); } + private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + return attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 2aaed13ec1..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -194,7 +194,9 @@ public abstract class AbstractConsumerService buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + link.setConfiguration(getReferencedEntityConfig(referencedEntityId)); + return link; + } + @Override public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { ObjectNode configNode = mapper.createObjectNode(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..ac94ade134 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -20,9 +20,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; @@ -57,4 +60,8 @@ public interface CalculatedFieldConfiguration { @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 685ca47719..8a7c2d8c03 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -824,10 +824,13 @@ message ProfileEntityMsgProto { message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CalculatedFieldEntityCtxIdProto links = 3; - repeated CalculatedFieldIdProto previousCalculatedFields = 4; - string scope = 5; - repeated TelemetryProto updatedTelemetry = 6; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldEntityCtxIdProto links = 6; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + string scope = 8; + repeated TelemetryProto updatedTelemetry = 9; } message CalculatedFieldEntityCtxIdProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 36bc3d038a..9c81d91f64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -240,23 +239,8 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - List links = buildCalculatedFieldLinks(tenantId, calculatedField); + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); - return cfConfig.getReferencedEntities().stream() - .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) - .map(referencedEntityId -> { - CalculatedFieldLink link = new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(referencedEntityId); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); - return link; - }) - .collect(Collectors.toList()); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 816fa1546c..bed6f2d3a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -29,7 +28,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); - List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 20081299e8..cdcffdd440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -57,7 +57,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); } @Override From 5203ef7422f32a0219a64470298e774c8b683ab8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:45:42 +0200 Subject: [PATCH 071/281] added calculated field state service --- .../service/cf/CalculatedFieldCache.java | 2 - .../cf/CalculatedFieldExecutionService.java | 2 - .../cf/DefaultCalculatedFieldCache.java | 51 +-- ...efaultCalculatedFieldExecutionService.java | 325 ++++++------------ .../server/service/cf/RocksDBService.java | 13 - .../queue/DefaultTbCoreConsumerService.java | 14 - .../server/common/util/ProtoUtils.java | 40 --- common/proto/src/main/proto/queue.proto | 43 +-- 8 files changed, 119 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 7394c95f08..bf5dc8d42f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,8 +36,6 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d1d459b9b..8ba1f6dfed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -27,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 71fc3eff67..f762ae3530 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,12 +36,12 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -72,14 +72,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); } @@ -90,41 +90,17 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); - } - - @Override - public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); - calculatedFieldFetchLock.lock(); - try { - List cfLinks = getCalculatedFieldLinks(calculatedFieldId); - if (cfLinks != null && !cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { - if (existingList == null) { - existingList = new ArrayList<>(); - } else if (!(existingList instanceof ArrayList)) { - existingList = new ArrayList<>(existingList); - } - existingList.add(link); - return existingList; - }); - }); - } - } finally { - calculatedFieldFetchLock.unlock(); - } + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override @@ -192,7 +168,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFields.put(calculatedFieldId, calculatedField); - entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -200,7 +176,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { configuration.getReferencedEntities().stream() .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) .forEach(referencedEntityId -> { - entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { @@ -210,13 +186,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); - try { - evict(calculatedFieldId); - addCalculatedField(tenantId, calculatedFieldId); - } finally { - calculatedFieldFetchLock.unlock(); - } + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd11c799c2..e54c1bd5c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -107,8 +108,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; -import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @@ -122,7 +121,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -148,8 +147,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> rocksDBService.getAll() - .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); + scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -223,10 +221,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - if (storedState != null) { - CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + if (restoredCtx != null) { states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { @@ -251,18 +248,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; } } if (cf != null) { + calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { @@ -312,12 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); + states.keySet().removeIf(ctxId -> { + if (ctxId.cfId().equals(calculatedFieldId)) { + stateService.removeState(ctxId); + return true; + } + return false; + }); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -366,7 +367,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -386,20 +386,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); - if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); @@ -411,14 +397,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, link, tpiStates); }); } else { - updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, link, tpiStates); } }); } + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, @@ -490,31 +490,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); - if (proto.getClear()) { - clearState(tenantId, calculatedFieldId, entityId); - return; - } - - List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .collect(Collectors.toCollection(ArrayList::new)); - Map argumentsMap = proto.getArgumentsMap().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); - } catch (Exception e) { - log.trace("Failed to process calculated field update state msg: [{}]", proto, e); - } - } - @Override public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { @@ -522,12 +497,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> clearState(tenantId, cfId, entityId)); - initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } else { + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); + } } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -539,37 +517,39 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(entityId, profileId) - .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntityByProfile(entityId, profileId, callback); + } } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } + + } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } } - private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - } + private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + states.remove(ctxId); + stateService.removeState(ctxId); } - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) + private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -607,65 +587,58 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - CompletableFuture updateFuture = new CompletableFuture<>(); + CompletableFuture updateFuture = new CompletableFuture<>(); - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentsMap)) { + calculatedFieldEntityCtx.setState(state); + stateService.persistState(entityCtxId, calculatedFieldEntityCtx); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } - updateFuture.complete(null); - }; + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + } + updateFuture.complete(null); + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + if (!allKeysPresent || requiresTsRollingUpdate) { + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } - return calculatedFieldEntityCtx; - }); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); - } + return calculatedFieldEntityCtx; + }); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { @@ -706,14 +679,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); - if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); - } - return links; - } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -777,89 +742,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); - } - if (previousCalculatedFieldIds != null) { - previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( - TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build() - )); - } - - log.info("Sending calculated field state msg from entityId [{}]", entityId); - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) - .setClear(true) - .build(); - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); - } - - private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( - TenantId tenantId, - CalculatedFieldId calculatedFieldId, - EntityId entityId - ) { - return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - } - - private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { - TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> - tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) - ); - argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - argumentProtoBuilder.setSingleValue( - TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(toObjectProto(singleValueArgumentEntry.getValue())) - .build() - ); - } - - return argumentProtoBuilder.build(); - } - - private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> - tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) - ); - return tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); } - ; - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { @@ -952,11 +838,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { + CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); + + if (state == null) { return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + return state; } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index d6b2980042..3aed65eced 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -27,7 +26,6 @@ import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.List; import java.util.Map; @Service @@ -59,17 +57,6 @@ public class RocksDBService { } } - public void deleteAll(List keys) { - try (WriteBatch batch = new WriteBatch()) { - for (String key : keys) { - batch.delete(key.getBytes(StandardCharsets.UTF_8)); - } - db.write(writeOptions, batch); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - public String get(String key) { try { byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6baa75b3ef..c598540ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -326,8 +326,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index d332bac64f..073f47d59b 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1316,46 +1316,6 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.ObjectProto toObjectProto(Object value) { - if (value == null) { - throw new IllegalArgumentException("Cannot convert null to ObjectProto"); - } - - TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); - - if (value instanceof String) { - builder.setStringValue((String) value); - } else if (value instanceof Integer) { - builder.setIntValue((Integer) value); - } else if (value instanceof Long) { - builder.setLongValue((Long) value); - } else if (value instanceof Double) { - builder.setDoubleValue((Double) value); - } else if (value instanceof Boolean) { - builder.setBoolValue((Boolean) value); - } else { - throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); - } - - return builder.build(); - } - - public static Object fromObjectProto(TransportProtos.ObjectProto proto) { - try { - return switch (proto.getValueCase()) { - case STRINGVALUE -> proto.getStringValue(); - case INTVALUE -> proto.getIntValue(); - case LONGVALUE -> proto.getLongValue(); - case DOUBLEVALUE -> proto.getDoubleValue(); - case BOOLVALUE -> proto.getBoolValue(); - case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", proto, e); - return null; - } - } - private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8a7c2d8c03..1036d5ba67 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,51 +841,11 @@ message CalculatedFieldEntityCtxIdProto { int64 entityIdLSB = 5; } -message CalculatedFieldStateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - string entityType = 5; - int64 entityIdMSB = 6; - int64 entityIdLSB = 7; - bool clear = 8; - repeated CalculatedFieldIdProto previousCalculatedFields = 9; - map arguments = 10; -} - message CalculatedFieldIdProto { int64 calculatedFieldIdMSB = 1; int64 calculatedFieldIdLSB = 2; } -message ArgumentEntryProto { - oneof entry_type { - TsRollingProto tsRecords = 1; - SingleValueProto singleValue = 2; - } -} - -message TsRollingProto { - map tsRecords = 1; -} - -message SingleValueProto { - int64 ts = 1; - ObjectProto value = 2; - int64 version = 3; -} - -message ObjectProto { - oneof value { - string stringValue = 1; - int32 intValue = 2; - int64 longValue = 3; - double doubleValue = 4; - bool boolValue = 5; - } -} - //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1632,7 +1592,6 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; - CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1681,6 +1640,8 @@ message ToRuleEngineMsg { repeated string relationTypes = 4; string failureMessage = 5; TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; + ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { From 77e99d15df2c2a3f412c48fce87151d2a473be7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:51:17 +0200 Subject: [PATCH 072/281] added service files --- .../cf/ctx/CalculatedFieldStateService.java | 30 +++++++++ .../cf/ctx/state/RocksDBStateService.java | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java new file mode 100644 index 0000000000..8bc5756f4e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import java.util.Map; + +public interface CalculatedFieldStateService { + + Map restoreStates(); + + CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + + void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + + void removeState(CalculatedFieldEntityCtxId ctxId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java new file mode 100644 index 0000000000..db8950804c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public class RocksDBStateService implements CalculatedFieldStateService { + + private final RocksDBService rocksDBService; + + @Override + public Map restoreStates() { + return rocksDBService.getAll().entrySet().stream() + .collect(Collectors.toMap( + entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + )); + } + + @Override + public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .orElse(null); + } + + @Override + public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId) { + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + +} From 6611f017c7eae1efb05b7bcf0bdf09d17b13ad34 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Jan 2025 16:58:26 +0200 Subject: [PATCH 073/281] changed Argument structure --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 8 ++ ...efaultCalculatedFieldExecutionService.java | 125 ++++++------------ .../cf/ctx/state/CalculatedFieldCtx.java | 15 ++- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 25 ++++ ...CalculatedFieldTelemetryUpdateRequest.java | 3 + ...alculatedFieldTimeSeriesUpdateRequest.java | 31 +++++ .../TbRuleEngineQueueConsumerManager.java | 5 + .../CalculatedFieldControllerTest.java | 2 +- .../data/cf/configuration/Argument.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 49 +++---- .../cf/configuration/ReferencedEntityKey.java | 30 +++++ .../dao/model/sql/CalculatedFieldEntity.java | 3 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 18 files changed, 195 insertions(+), 122 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index bf5dc8d42f..8730aeeedf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -38,6 +38,8 @@ public interface CalculatedFieldCache { CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index f762ae3530..868001d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -61,6 +61,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @@ -126,6 +127,13 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return ctx; } + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .toList(); + } + @Override public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { Set entities = profileEntities.get(entityProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e54c1bd5c9..7c17d54649 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -273,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -341,30 +339,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { try { - EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + TenantId tenantId = request.getTenantId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); } @@ -375,13 +373,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } } @@ -393,56 +390,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); }); } else { - processCalculatedFieldLink(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); } }); } - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - - private void mapAndProcessUpdatedTelemetry(TenantId tenantId, - EntityId entityId, - CalculatedFieldId calculatedFieldId, - CalculatedFieldTelemetryUpdateRequest request, - CalculatedFieldLinkConfiguration linkConfiguration) { - Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); } } - private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { - return kvEntries.stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), - entry -> entry, - (v1, v2) -> v1 - )); - } - @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { @@ -454,40 +427,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - CalculatedFieldLinkConfiguration linkConfiguration - = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); - - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - } - default -> - updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); - } + updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -533,8 +492,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } - - } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } @@ -616,11 +573,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) @@ -696,7 +653,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); + EntityId argumentEntityId = argument.getRefEntityId(); EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; @@ -704,17 +661,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { + return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); case TS_LATEST -> transformSingleValueArgument( Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); }; @@ -736,7 +693,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); @@ -826,7 +783,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); + String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index d54a3220ed..e17a1a61f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -22,13 +22,16 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { @@ -38,7 +41,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final List argKeys; + private final Map, String> referencedEntityKeys; + private final List argNames; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -51,7 +55,12 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); - this.argKeys = new ArrayList<>(arguments.keySet()); + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; @@ -69,7 +78,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - argKeys.toArray(String[]::new) + argNames.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index de7c514786..0421055fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = ctx.getArgKeys().stream() + Object[] args = ctx.getArgNames().stream() .map(key -> arguments.get(key).getValue()) .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index a83cc0fc25..6050370fd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -20,11 +20,16 @@ import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,4 +60,24 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel }; } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + + String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + + if (argName != null) { + mappedKvEntries.put(argName, entry); + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 29ee899ec9..f85117dc41 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; import java.util.Map; @@ -36,4 +37,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); + Map getMappedTelemetry(CalculatedFieldCtx ctx); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 507daf386e..646145a46e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -19,11 +19,16 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,4 +53,30 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return linkConfiguration.getTimeSeries(); } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + + if (argTsLatestName != null) { + mappedKvEntries.put(argTsLatestName, entry); + } else { + ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + + if (argTsRollingName != null) { + mappedKvEntries.put(argTsRollingName, entry); + } + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 243a3adbf7..83ab90e3bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -179,6 +180,10 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager getReferencedEntities() { return arguments.values().stream() - .map(Argument::getEntityId) + .map(Argument::getRefEntityId) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -69,24 +70,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { - Argument targetArgument = entry.getValue(); - String argumentKey = entry.getKey(); + ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); + String argumentName = entry.getKey(); - switch (targetArgument.getType()) { + switch (refEntityKey.getType()) { case ATTRIBUTE -> { - switch (targetArgument.getScope()) { + switch (refEntityKey.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(refEntityKey.getKey(), argumentName); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(refEntityKey.getKey(), argumentName); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(refEntityKey.getKey(), argumentName); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(refEntityKey.getKey(), argumentName); } }); @@ -118,7 +119,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel ObjectNode argumentsNode = configNode.putObject("arguments"); arguments.forEach((key, argument) -> { ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); + EntityId referencedEntityId = argument.getRefEntityId(); if (referencedEntityId != null) { argumentNode.put("entityType", referencedEntityId.getEntityType().name()); argumentNode.put("entityId", referencedEntityId.getId().toString()); @@ -126,9 +127,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityType", entityType.name()); argumentNode.put("entityId", entityId.toString()); } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", String.valueOf(argument.getType())); - argumentNode.put("scope", String.valueOf(argument.getScope())); +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", String.valueOf(argument.getType())); +// argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); @@ -165,19 +166,19 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { String referencedEntityType = argumentNode.get("entityType").asText(); UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } - argument.setKey(argumentNode.get("key").asText()); +// argument.setRefEntityKey(argumentNode.get("key").asText()); JsonNode type = argumentNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - argument.setType(ArgumentType.valueOf(type.asText())); - } - JsonNode scope = argumentNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - argument.setScope(AttributeScope.valueOf(scope.asText())); - } +// if (type != null && !type.isNull() && !type.asText().equals("null")) { +// argument.setType(ArgumentType.valueOf(type.asText())); +// } +// JsonNode scope = argumentNode.get("scope"); +// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { +// argument.setScope(AttributeScope.valueOf(scope.asText())); +// } if (argumentNode.hasNonNull("defaultValue")) { argument.setDefaultValue(argumentNode.get("defaultValue").asText()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..b49495d959 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6aaaf05836..d1352a26b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -22,6 +22,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -95,6 +96,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); +// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { @@ -112,6 +114,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); +// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 9a0b9222f0..f38722c9c1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -887,7 +887,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a8f7a2383..a9a70d8bc8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -156,7 +156,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index d0ee833261..236b159ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -382,7 +382,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 5b060ae145..cca30e6fbc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1225,7 +1225,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); From 3908105ac5e4cba45100a2020beb61db1a1e34c2 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 13 Jan 2025 13:23:41 +0200 Subject: [PATCH 074/281] fix_bug: tbel, in the isDecimal method. The mantissa format includes numbers written in exponential form --- .../java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- .../org/thingsboard/script/api/tbel/TbUtilsTest.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index fae91d14ca..ab1aef979a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1299,7 +1299,7 @@ public class TbUtils { if (str == null || str.isEmpty()) { return -1; } - return str.matches("[+-]?\\d+(\\.\\d+)?") ? DEC_RADIX : -1; + return str.matches("[+-]?\\d+(\\.\\d+)?([eE][+-]?\\d+)?") ? DEC_RADIX : -1; } public static int isHexadecimal(String str) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 21ee8538a0..4ac387b258 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -442,8 +442,9 @@ public class TbUtilsTest { @Test public void parsDouble() { - String doubleValStr = "1729.1729"; - Assertions.assertEquals(java.util.Optional.of(doubleVal).get(), TbUtils.parseDouble(doubleValStr)); + String doubleValStr = "1.1428250947E8"; + Assertions.assertEquals(Double.parseDouble(doubleValStr), TbUtils.parseDouble(doubleValStr)); + doubleValStr = "1729.1729"; Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseHexToDouble(longValHex))); Assertions.assertEquals(0, Double.compare(doubleValRev, TbUtils.parseHexToDouble(longValHex, false))); Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseBigEndianHexToDouble(longValHex))); @@ -930,7 +931,13 @@ public class TbUtilsTest { @Test public void isDecimal_Test() { Assertions.assertEquals(10, TbUtils.isDecimal("4567039")); + Assertions.assertEquals(10, TbUtils.isDecimal("1.1428250947E8")); + Assertions.assertEquals(10, TbUtils.isDecimal("123.45")); + Assertions.assertEquals(10, TbUtils.isDecimal("-1.23E-4")); + Assertions.assertEquals(10, TbUtils.isDecimal("1E5")); Assertions.assertEquals(-1, TbUtils.isDecimal("C100110")); + Assertions.assertEquals(-1, TbUtils.isDecimal("abc")); + Assertions.assertEquals(-1, TbUtils.isDecimal(null)); } @Test From f0a36d500883a382e540e861b0a36be20e5e9fbf Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:03:50 +0200 Subject: [PATCH 075/281] fixed tests --- .../BaseCalculatedFieldConfiguration.java | 129 +----------------- .../CalculatedFieldConfiguration.java | 6 - .../ScriptCalculatedFieldConfiguration.java | 12 -- .../SimpleCalculatedFieldConfiguration.java | 12 -- .../dao/model/sql/CalculatedFieldEntity.java | 16 +-- ...efaultNativeCalculatedFieldRepository.java | 11 +- .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 7 +- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 10 files changed, 20 insertions(+), 194 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8cab602683..87cece0419 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -15,48 +15,25 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.stream.Collectors; @Data public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - @JsonIgnore - private final ObjectMapper mapper = new ObjectMapper(); - protected Map arguments; protected String expression; protected Output output; - public BaseCalculatedFieldConfiguration() { - } - - public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { -// BaseCalculatedFieldConfiguration calculatedFieldConfig = mapper.convertValue(config, BaseCalculatedFieldConfiguration.class); - BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); - this.arguments = calculatedFieldConfig.getArguments(); - this.expression = calculatedFieldConfig.getExpression(); - this.output = calculatedFieldConfig.getOutput(); - } - @Override public List getReferencedEntities() { return arguments.values().stream() @@ -70,7 +47,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId() != null && entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); String argumentName = entry.getKey(); @@ -112,108 +89,4 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return link; } - @Override - public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { - ObjectNode configNode = mapper.createObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - arguments.forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getRefEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", String.valueOf(argument.getType())); -// argumentNode.put("scope", String.valueOf(argument.getScope())); - argumentNode.put("defaultValue", argument.getDefaultValue()); - argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); - }); - - if (expression != null) { - configNode.put("expression", expression); - } - - if (output != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("name", output.getName()); - outputNode.put("type", String.valueOf(output.getType())); - if (output.getScope() != null) { - outputNode.put("scope", String.valueOf(output.getScope())); - } - } - - return configNode; - } - - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null || !config.isObject()) { - return null; - } - - Map arguments = new HashMap<>(); - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - Argument argument = new Argument(); - if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } -// argument.setRefEntityKey(argumentNode.get("key").asText()); - JsonNode type = argumentNode.get("type"); -// if (type != null && !type.isNull() && !type.asText().equals("null")) { -// argument.setType(ArgumentType.valueOf(type.asText())); -// } -// JsonNode scope = argumentNode.get("scope"); -// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { -// argument.setScope(AttributeScope.valueOf(scope.asText())); -// } - if (argumentNode.hasNonNull("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); - } - if (argumentNode.hasNonNull("limit")) { - argument.setLimit(argumentNode.get("limit").asInt()); - } - if (argumentNode.hasNonNull("timeWindow")) { - argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); - } - arguments.put(key, argument); - }); - } - this.setArguments(arguments); - - JsonNode expressionNode = config.get("expression"); - if (expressionNode != null && expressionNode.isTextual()) { - this.setExpression(expressionNode.asText()); - } - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - Output output = new Output(); - output.setName(outputNode.get("name").asText()); - JsonNode type = outputNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - output.setType(OutputType.valueOf(type.asText())); - } - JsonNode scope = outputNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - output.setScope(AttributeScope.valueOf(scope.asText())); - } - this.setOutput(output); - } - - return this; - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index ac94ade134..8f56bf491d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -29,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; -import java.util.UUID; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -57,9 +54,6 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); - @JsonIgnore - JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); - List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java index a24328b4c9..017fc5a485 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public ScriptCalculatedFieldConfiguration() { - super(); - } - - public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index af11d2f5d8..6312c3e1db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public SimpleCalculatedFieldConfiguration() { - super(); - } - - public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index d1352a26b8..a0157cde66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,12 +23,9 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -96,8 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); -// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -113,8 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); -// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -122,11 +117,4 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - return switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index a5a2743f26..bb88982e3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +88,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(type, configuration, entityType, entityId)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); @@ -135,11 +133,4 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - return switch (type) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index f38722c9c1..aed1621e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -885,9 +886,9 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedAsset.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a9a70d8bc8..6dd84714f8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -154,9 +155,9 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 236b159ddb..6e57279f38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -380,9 +381,9 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedCustomer.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index cca30e6fbc..959825e113 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1223,9 +1224,9 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(device.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From d9ccc8118cf316389e3c284e468b4236118462bb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:52:06 +0200 Subject: [PATCH 076/281] fixed calculated field controller test --- .../server/controller/CalculatedFieldControllerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 565321afbc..b1c7547251 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -141,9 +142,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From 0f34f131c991012f9c499e36475064f0c4a7b97c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 21 Jan 2025 14:30:27 +0200 Subject: [PATCH 077/281] Return TimeseriesSaveResult --- .../DefaultTelemetrySubscriptionService.java | 35 +++++----- .../telemetry/InternalTelemetryService.java | 3 +- .../dao/timeseries/TimeseriesService.java | 9 +-- .../common/data/kv/TimeseriesSaveResult.java | 26 +++++++ .../dao/timeseries/BaseTimeseriesService.java | 69 ++++++++----------- 5 files changed, 79 insertions(+), 63 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..e7dfbcdca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -126,10 +127,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; if (sysTenant || request.isOnlyLatest() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); - ListenableFuture future = saveTimeseriesInternal(request); + ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -137,27 +137,27 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { + public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - ListenableFuture saveFuture; + ListenableFuture resultFuture; if (request.isOnlyLatest()) { - saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); + resultFuture = tsService.saveLatest(tenantId, entityId, request.getEntries()); } else if (request.isSaveLatest()) { - saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); } else { - saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(saveFuture, request.getCallback()); - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); + addMainCallback(resultFuture, request.getCallback()); + addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); - return saveFuture; + addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + return resultFuture; } @Override @@ -329,19 +329,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override - public void onSuccess(Integer result) { - if (!sysTenant && result != null && result > 0) { - apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + public void onSuccess(TimeseriesSaveResult result) { + Integer dataPoints = result.getDataPoints(); + if (!sysTenant && dataPoints != null && dataPoints > 0) { + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, dataPoints); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 8e45b84a75..8a236bf002 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,13 +21,14 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** * Created by ashvayka on 27.03.18. */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); + ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); void saveAttributesInternal(AttributesSaveRequest request); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 49918f3823..542f77445b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -44,13 +45,13 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java new file mode 100644 index 0000000000..edec063be2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +import java.util.List; + +@Data(staticConstructor = "of") +public class TimeseriesSaveResult { + private final Integer dataPoints; + private final List versions; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 756b73d88b..6b4a3c917e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.entityview.EntityViewService; @@ -156,60 +157,48 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); - List> futures = new ArrayList<>(INSERTS_PER_ENTRY); - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + return doSave(tenantId, entityId, List.of(tsKvEntry), 0L, true, true); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, true); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, true, true); } @Override - public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, false); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; - List> futures = new ArrayList<>(tsKvEntries.size() * inserts); - for (TsKvEntry tsKvEntry : tsKvEntries) { - if (saveLatest) { - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } else { - saveWithoutLatestAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } - } - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, false, true); } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); - for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); - } - return Futures.allAsList(futures); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + return doSave(tenantId, entityId, tsKvEntries, 0L, true, false); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); - } - - private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - } - - private void doSaveAndRegisterFuturesFor(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest, boolean saveTs) { + if (saveTs && entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } - futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); - futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + List> tsFutures = saveTs ? new ArrayList<>(tsKvEntries.size() * INSERTS_PER_ENTRY_WITHOUT_LATEST) : null; + List> latestFutures = saveLatest ? new ArrayList<>(tsKvEntries.size()) : null; + for (TsKvEntry tsKvEntry : tsKvEntries) { + if (saveTs) { + tsFutures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); + tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + } + if (saveLatest) { + latestFutures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + } + } + ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); + ListenableFuture> versionsFuture = saveLatest ? Futures.allAsList(latestFutures) : Futures.immediateFuture(null); + return Futures.whenAllComplete(dpsFuture, versionsFuture).call(() -> { + Integer dataPoints = Futures.getUnchecked(dpsFuture); + List versions = Futures.getUnchecked(versionsFuture); + return TimeseriesSaveResult.of(dataPoints, versions); + }, MoreExecutors.directExecutor()); } private List updateQueriesForEntityView(EntityView entityView, List queries) { From 9ef68584c9aecb19659496210cf914a7eb5f81b5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 09:40:12 +0200 Subject: [PATCH 078/281] updated getMappedTelemetry method --- ...efaultCalculatedFieldExecutionService.java | 19 ++++++++++--------- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 15 ++------------- ...CalculatedFieldTelemetryUpdateRequest.java | 5 +---- ...alculatedFieldTimeSeriesUpdateRequest.java | 13 +++---------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7c17d54649..84a21ab315 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -271,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) + .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -375,9 +375,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } @@ -406,9 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); @@ -427,13 +428,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } catch (Exception e) { @@ -654,7 +655,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) + EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e17a1a61f9..cb4052b7df 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -57,7 +57,7 @@ public class CalculatedFieldCtx { this.arguments = configuration.getArguments(); this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( - entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), Map.Entry::getKey )); this.argNames = new ArrayList<>(arguments.keySet()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 6050370fd2..d2eb31cd6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -52,16 +51,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return switch (scope) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); - }; - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -70,7 +60,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + String argName = referencedKeys.get(new TbPair<>(referencedEntityId, referencedEntityKey)); if (argName != null) { mappedKvEntries.put(argName, entry); @@ -79,5 +69,4 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel return mappedKvEntries; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index f85117dc41..3f7250f4ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,8 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); - - Map getMappedTelemetry(CalculatedFieldCtx ctx); + Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 646145a46e..a5637c8cfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -49,12 +48,7 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return linkConfiguration.getTimeSeries(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -62,13 +56,13 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe String key = entry.getKey(); ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + String argTsLatestName = referencedKeys.get(new TbPair<>(referencedEntityId, tsLatestKey)); if (argTsLatestName != null) { mappedKvEntries.put(argTsLatestName, entry); } else { ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + String argTsRollingName = referencedKeys.get(new TbPair<>(referencedEntityId, tsRollingKey)); if (argTsRollingName != null) { mappedKvEntries.put(argTsRollingName, entry); @@ -78,5 +72,4 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return mappedKvEntries; } - } From 6b9d374a5f2957d14a3722c2e6a6459da211db11 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jan 2025 12:23:42 +0200 Subject: [PATCH 079/281] Tmp commit for merge --- .../cf/CalculatedFieldExecutionService.java | 11 +++ .../TbCalculatedFieldConsumerService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 18 +++-- .../src/main/resources/thingsboard.yml | 12 ++- .../server/common/data/DataConstants.java | 2 + .../server/common/msg/queue/ServiceType.java | 3 +- common/proto/src/main/proto/queue.proto | 73 +++++++------------ .../queue/discovery/HashPartitionService.java | 12 ++- .../discovery/event/PartitionChangeEvent.java | 4 + 9 files changed, 86 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 8ba1f6dfed..e18c8b4119 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -21,6 +21,17 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { + /** + * Push incoming telemetry to the CF processing queue for async processing. + * @param request - telemetry request; + * @param callback - callback to be executed when the message is ack by the queue. + */ + void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + + void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + + /* ===================================================== */ + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..387bdd7143 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -0,0 +1,8 @@ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface TbCalculatedFieldConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 8773564e5d..dcb72b8dd0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.telemetry; +import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -128,8 +129,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -148,7 +148,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - + // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. + saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { + @Override + public ListenableFuture apply(Integer input) throws Exception { + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); + return input; + } + }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -326,19 +333,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override public void onSuccess(Integer result) { if (!sysTenant && result != null && result > 0) { apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); + } }; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3014e1448b..5151bc019b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1692,7 +1692,6 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" - vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" @@ -1739,6 +1738,17 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + calculated-fields: + # Topic name for Calculated Field (CF) tasks + topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + # Interval in milliseconds to poll messages by CF (Rule Engine) microservices + poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + # Amount of partitions used by CF microservices + partitions: "${TB_QUEUE_CF_PARTITIONS:10}" + # Timeout for processing a message pack by CF microservices + pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + # Enable/disable a separate consumer per partition for CF queue + consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 56a5e135f8..77a7c4a781 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -145,4 +145,6 @@ public class DataConstants { public static final String EDGE_QUEUE_NAME = "Edge"; public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; + public static final String CF_QUEUE_NAME = "CalculatedFields"; + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f3a0e47d09..022c46bcc6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + TB_CF_ENGINE("TB Calculated Fields Engine"); private final String label; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1036d5ba67..4e719a02df 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,18 +183,6 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } -message AttributeKvProto { - AttributeKey key = 1; - AttributeValueProto value = 2; -} - -message TelemetryProto { - oneof proto { - AttributeKvProto attrKv = 1; - TsKvProto tsKv = 2; - } -} - message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -785,17 +773,7 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - -message EntityProfileUpdateMsgProto { +message CalculatedFieldEntityUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -806,31 +784,26 @@ message EntityProfileUpdateMsgProto { int64 oldProfileIdLSB = 8; int64 newProfileIdMSB = 9; int64 newProfileIdLSB = 10; + bool added = 11; + bool updated = 12; + bool deleted = 13; } -message ProfileEntityMsgProto { +message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; int64 entityIdMSB = 4; int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 profileIdMSB = 7; - int64 profileIdLSB = 8; - bool added = 9; - bool deleted = 10; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + repeated TsKvProto tsData = 9; + AttributeScopeProto scope = 10; + repeated AttributeValueProto attrData = 11; } -message TelemetryUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - repeated CalculatedFieldEntityCtxIdProto links = 6; - repeated CalculatedFieldIdProto previousCalculatedFields = 7; - string scope = 8; - repeated TelemetryProto updatedTelemetry = 9; +message CalculatedFieldLinkedTelemetryMsgProto { + CalculatedFieldTelemetryMsgProto msg = 1; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1589,9 +1562,8 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; - CalculatedFieldMsgProto calculatedFieldMsg = 53; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - ProfileEntityMsgProto profileEntityMsg = 55; +// CalculatedFieldMsgProto calculatedFieldMsg = 53; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1611,8 +1583,8 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; - ProfileEntityMsgProto profileEntityMsg = 52; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; +// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ @@ -1632,6 +1604,16 @@ message ToEdgeEventNotificationMsg { EdgeEventMsgProto edgeEventMsg = 1; } +message ToCalculatedFieldMsg { + CalculatedFieldTelemetryMsgProto telemetryMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; +} + +message ToCalculatedFieldNotificationMsg { + ComponentLifecycleMsgProto componentLifecycle = 1; + CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; @@ -1639,9 +1621,6 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; - TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; - ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 37e519e3f2..53bdc78c93 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,8 +51,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; -import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.*; @Service @Slf4j @@ -62,6 +61,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; + @Value("${queue.calculated-fields.topic}") + private String cfTopic; + @Value("${queue.calculated-fields.partitions:10}") + private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") @@ -108,10 +111,15 @@ public class HashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); + QueueKey coreKey = new QueueKey(ServiceType.TB_CORE); partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + partitionSizesMap.put(cfKey, cfPartitions); + partitionTopicsMap.put(cfKey, cfTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); partitionTopicsMap.put(vcKey, vcTopic); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 88ceb4aa08..3bb0c56f9a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -52,6 +52,10 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getCalculatedFieldsPartitions() { + return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { return partitionsMap.entrySet() .stream() From c047d5f4f065d17ea80935bb4f3ebb9eae518c80 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 17:01:28 +0200 Subject: [PATCH 080/281] added debug event entity --- .../server/actors/ActorSystemContext.java | 41 ++++++ .../actors/ruleChain/DefaultTbContext.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 13 +- .../common/data/cf/CalculatedField.java | 23 ++- .../data/event/CalculatedFieldDebugEvent.java | 95 ++++++++++++ .../CalculatedFieldDebugEventFilter.java | 50 +++++++ .../server/common/data/event/EventType.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + .../server/dao/event/BaseEventService.java | 6 + .../server/dao/model/ModelConstants.java | 5 + .../sql/CalculatedFieldDebugEventEntity.java | 104 ++++++++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 7 + .../CalculatedFieldDebugEventRepository.java | 135 ++++++++++++++++++ .../dao/sql/event/DedicatedJpaEventDao.java | 5 +- .../dao/sql/event/EventInsertRepository.java | 29 ++++ .../server/dao/sql/event/JpaBaseEventDao.java | 47 ++++++ .../main/resources/sql/schema-entities.sql | 17 +++ 17 files changed, 570 insertions(+), 13 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 71fa8fa958..e6f541090c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -42,10 +42,12 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -125,6 +127,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -157,6 +160,18 @@ public class ActorSystemContext { } }; + private static final FutureCallback CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Calculated Field", th); + } + }; + private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); public ConcurrentMap getDebugPerTenantLimits() { @@ -723,6 +738,32 @@ public class ActorSystemContext { } } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { + if (checkLimits(tenantId, tbMsg, error)) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(tbMsg.getOriginator()) + .msgId(tbMsg.getId()) + .msgType(tbMsg.getType()) + .arguments(JacksonUtil.toString(arguments)) + .result(tbMsg.getData()); + + if (error != null) { + event.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + } + } + private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 421e4efb26..54e32446bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; @@ -64,7 +65,6 @@ import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.msg.TbActorMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..a365e2bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -71,6 +70,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -122,6 +122,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; + private final EventService eventService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -253,7 +254,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; @@ -301,6 +301,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { + calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); callback.onSuccess(); shouldReinit = false; } @@ -329,13 +330,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + return entityIdChanged || typeChanged || argumentsChanged; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index e626c9d3d2..f4b92b3802 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; @@ -22,11 +24,13 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -36,7 +40,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { private static final long serialVersionUID = 4491966747773381420L; @@ -50,6 +54,11 @@ public class CalculatedField extends BaseData implements HasN @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") private String name; + @Deprecated + @Schema(description = "Enable/disable debug. ", example = "false", deprecated = true) + private boolean debugMode; + @Schema(description = "Debug settings object.") + private DebugSettings debugSettings; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; @Schema(implementation = SimpleCalculatedFieldConfiguration.class) @@ -109,4 +118,16 @@ public class CalculatedField extends BaseData implements HasN .toString(); } + // Getter is ignored for serialization + @JsonIgnore + public boolean isDebugMode() { + return debugMode; + } + + // Setter is annotated for deserialization + @JsonSetter + public void setDebugMode(boolean debugMode) { + this.debugMode = debugMode; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java new file mode 100644 index 0000000000..e0599db358 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldDebugEvent extends Event { + + private static final long serialVersionUID = -7091690784759639853L; + + @Builder + private CalculatedFieldDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + CalculatedFieldId calculatedFieldId, EntityId eventEntity, UUID msgId, + String msgType, String arguments, String result, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.calculatedFieldId = calculatedFieldId; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.arguments = arguments; + this.result = result; + this.error = error; + } + + @Getter + private final CalculatedFieldId calculatedFieldId; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + @Setter + private String arguments; + @Getter + @Setter + private String result; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("calculatedFieldId", calculatedFieldId.toString()); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "arguments", arguments); + putNotNull(json, "result", result); + putNotNull(json, "error", error); + return eventInfo; + } + + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java new file mode 100644 index 0000000000..839583ef6c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; + +@Data +@EqualsAndHashCode(callSuper = true) +@Schema +public class CalculatedFieldDebugEventFilter extends DebugEventFilter { + + @Schema(description = "String value representing the calculated field id in the event body", example = "ccbfa2fe-c8f5-45d8-bb37-6b61a6e02833") + protected String calculatedFieldId; + @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") + protected String entityId; + @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @Schema(description = "String value representing the message id in the rule engine", example = "dcf44612-2ce4-4e5d-b462-ebb9c5628228") + protected String msgId; + @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + protected String msgType; + + @Override + public EventType getEventType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(calculatedFieldId) || !StringUtils.isEmpty(entityId) + || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 6f98e1537f..46d0e49370 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -22,7 +22,8 @@ public enum EventType { LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), - DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true), + DEBUG_CALCULATED_FIELD("cf_debug_event", "DEBUG_CALCULATED_FIELD", true); @Getter private final String table; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 9c81d91f64..26ed4134cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -64,6 +64,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 5314dcd405..dc57b07f72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -88,6 +89,11 @@ public class BaseEventService implements EventService { ErrorEvent eEvent = (ErrorEvent) event; truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); break; + case DEBUG_CALCULATED_FIELD: + CalculatedFieldDebugEvent cfEvent = (CalculatedFieldDebugEvent) event; + truncateField(cfEvent, CalculatedFieldDebugEvent::getArguments, CalculatedFieldDebugEvent::setArguments); + truncateField(cfEvent, CalculatedFieldDebugEvent::getResult, CalculatedFieldDebugEvent::setResult); + truncateField(cfEvent, CalculatedFieldDebugEvent::getError, CalculatedFieldDebugEvent::setError); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 6bd6a3a384..0cc2f03d2e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -373,6 +373,7 @@ public class ModelConstants { public static final String STATS_EVENT_TABLE_NAME = "stats_event"; public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME = "cf_debug_event"; public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; @@ -397,6 +398,10 @@ public class ModelConstants { public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String EVENT_CALCULATED_FIELD_ID_COLUMN_NAME = "cf_id"; + public static final String EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME = "e_args"; + public static final String EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME = "e_result"; + public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_SETTINGS = "debug_settings"; public static final String SINGLETON_MODE = "singleton_mode"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java new file mode 100644 index 0000000000..57849ae3ea --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class CalculatedFieldDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_CALCULATED_FIELD_ID_COLUMN_NAME) + private UUID calculatedFieldId; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME) + private String arguments; + @Column(name = EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME) + private String result; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public CalculatedFieldDebugEventEntity(CalculatedFieldDebugEvent event) { + super(event); + if (event.getCalculatedFieldId() != null) { + this.calculatedFieldId = event.getCalculatedFieldId().getId(); + } + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.arguments = event.getArguments(); + this.result = event.getResult(); + this.error = event.getError(); + } + + @Override + public CalculatedFieldDebugEvent toData() { + var builder = CalculatedFieldDebugEvent.builder() + .id(id) + .tenantId(TenantId.fromUUID(tenantId)) + .ts(ts) + .serviceId(serviceId) + .entityId(entityId) + .msgId(msgId) + .msgType(msgType) + .arguments(arguments) + .result(result) + .error(error); + if (calculatedFieldId != null) { + builder.calculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + } + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index a0157cde66..64c8e8d5b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -45,6 +46,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_T import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @Data @EqualsAndHashCode(callSuper = true) @@ -77,6 +79,9 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem @Column(name = CALCULATED_FIELD_VERSION) private Long version; + @Column(name = DEBUG_SETTINGS) + private String debugSettings; + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) private UUID externalId; @@ -95,6 +100,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); + this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); } @@ -111,6 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java new file mode 100644 index 0000000000..c0bd21ca74 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM cf_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java index 9b7af5e7f7..7bf9b426e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java @@ -36,10 +36,11 @@ public class DedicatedJpaEventDao extends JpaBaseEventDao { RuleNodeDebugEventRepository ruleNodeDebugEventRepository, RuleChainDebugEventRepository ruleChainDebugEventRepository, ScheduledLogExecutorComponent logExecutor, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CalculatedFieldDebugEventRepository cfDebugEventRepository) { super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository, errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository, - ruleChainDebugEventRepository, logExecutor, statsFactory); + ruleChainDebugEventRepository, logExecutor, statsFactory, cfDebugEventRepository); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 962be57892..9307b9e9be 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -81,6 +82,9 @@ public class EventInsertRepository { insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + + " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } public void save(List entities) { @@ -107,6 +111,8 @@ public class EventInsertRepository { return getRuleNodeEventSetter(events); case DEBUG_RULE_CHAIN: return getRuleChainEventSetter(events); + case DEBUG_CALCULATED_FIELD: + return getCalculatedFieldEventSetter(events); default: throw new RuntimeException(eventType + " support is not implemented!"); } @@ -206,6 +212,29 @@ public class EventInsertRepository { }; } + private BatchPreparedStatementSetter getCalculatedFieldEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + CalculatedFieldDebugEvent event = (CalculatedFieldDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutUUID(ps, 6, event.getCalculatedFieldId().getId()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getArguments()); + safePutString(ps, 12, event.getResult()); + safePutString(ps, 13, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { if (value != null) { ps.setString(parameterIdx, replaceNullChars(value)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index b8d5083402..e3c1b37536 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -72,6 +73,7 @@ public class JpaBaseEventDao implements EventDao { private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final ScheduledLogExecutorComponent logExecutor; private final StatsFactory statsFactory; + private final CalculatedFieldDebugEventRepository calculatedFieldDebugEventRepository; @Value("${sql.events.batch_size:10000}") private int batchSize; @@ -110,6 +112,7 @@ public class JpaBaseEventDao implements EventDao { repositories.put(EventType.ERROR, errorEventRepository); repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); + repositories.put(EventType.DEBUG_CALCULATED_FIELD, calculatedFieldDebugEventRepository); } @PreDestroy @@ -158,6 +161,8 @@ public class JpaBaseEventDao implements EventDao { return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); case STATS: return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + case DEBUG_CALCULATED_FIELD: + return findEventByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, pageLink); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -193,6 +198,8 @@ public class JpaBaseEventDao implements EventDao { case STATS: removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); break; + case DEBUG_CALCULATED_FIELD: + removeEventsByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, startTime, endTime); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -286,6 +293,27 @@ public class JpaBaseEventDao implements EventDao { ); } + private PageData findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + calculatedFieldDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); + } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { ruleChainDebugEventRepository.removeEvents( tenantId, @@ -360,6 +388,25 @@ public class JpaBaseEventDao implements EventDao { ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + calculatedFieldDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 51cc66c4ac..a6d1e8800d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -934,6 +934,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration_version int DEFAULT 0, configuration varchar(1000000), version BIGINT DEFAULT 1, + debug_settings varchar(1024), external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) @@ -949,3 +950,19 @@ CREATE TABLE IF NOT EXISTS calculated_field_link ( configuration varchar(10000), CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE ); + +CREATE TABLE IF NOT EXISTS cf_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar, + cf_id uuid NOT NULL, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_args varchar, + e_result varchar, + e_error varchar +) PARTITION BY RANGE (ts); From 069725a2b95dd22f8e7ab59250904c8d8897d11f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 10:43:26 +0200 Subject: [PATCH 081/281] WIP Refactoring of the cluster mode --- ...faultTbCalculatedFieldConsumerService.java | 245 ++++++++++++++++++ .../DefaultTbRuleEngineConsumerService.java | 5 + .../DefaultTelemetrySubscriptionService.java | 14 +- .../src/main/resources/thingsboard.yml | 16 +- .../server/common/util/ProtoUtils.java | 30 +-- common/proto/src/main/proto/queue.proto | 29 +++ .../queue/discovery/HashPartitionService.java | 15 +- .../server/queue/discovery/TopicService.java | 12 +- .../discovery/event/PartitionChangeEvent.java | 5 +- .../InMemoryMonolithQueueFactory.java | 34 +++ .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbRuleEngineQueueFactory.java | 24 +- .../TbQueueCalculatedFieldSettings.java | 35 +++ 13 files changed, 430 insertions(+), 45 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..85ee01ac5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -0,0 +1,245 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.List; +import java.util.UUID; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { + + @Value("${queue.calculated_fields.poll_interval}") + private long pollInterval; + @Value("${queue.calculated_fields.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + @Value("${queue.calculated_fields.pool_size:8}") + private int poolSize; + + private final TbRuleEngineQueueFactory queueFactory; + + private final CalculatedFieldExecutionService calculatedFieldExecutionService; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; + + private volatile ListeningExecutorService calculatedFieldsExecutor; + + public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, + ActorSystemContext actorContext, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService, + CalculatedFieldExecutionService calculatedFieldExecutionService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, + eventPublisher, jwtSettingsService); + this.queueFactory = tbQueueFactory; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; + } + + @PostConstruct + public void init() { + super.init("tb-cf"); + this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + + this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(new QueueKey(ServiceType.TB_CORE)) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor(this::processMsgs) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + if (calculatedFieldsExecutor != null) { + calculatedFieldsExecutor.shutdownNow(); + } + } + + @Override + protected void startConsumers() { + super.startConsumers(); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + mainConsumer.update(event.getCalculatedFieldsPartitions()); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { + + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollInterval; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected int getMgmtThreadPoolSize() { + return Math.max(Runtime.getRuntime().availableProcessors(), 4); + } + + @Override + protected TbQueueConsumer> createNotificationsConsumer() { + return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCalculatedFieldNotificationMsg notification = msg.getValue(); + + callback.onSuccess(); + } + +// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); +// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); +// } +// +// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { +// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); +// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); +// boolean added = profileEntityMsg.getAdded(); +// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); +// if (added) { +// entitiesByProfile.add(entityId); +// } else { +// entitiesByProfile.remove(entityId); +// } +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { +// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); +// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { +// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { + return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); + } + + @Override + protected void stopConsumers() { + super.stopConsumers(); + mainConsumer.stop(); + mainConsumer.awaitStop(); + } + + @Data(staticConstructor = "of") + public static class CalculatedFieldQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 7d4d975cb4..5e07e33a9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -63,6 +63,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @Service @TbRuleEngineComponent @Slf4j @@ -107,6 +109,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { + if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + return; + } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index dcb72b8dd0..fdf3ed6c50 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -149,13 +149,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. - saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { - @Override - public ListenableFuture apply(Integer input) throws Exception { - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); - return input; - } - }); +// saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { +// @Override +// public ListenableFuture apply(Integer input) throws Exception { +// calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); +// return input; +// } +// }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5151bc019b..94c1bef71c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1738,17 +1738,21 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" - calculated-fields: - # Topic name for Calculated Field (CF) tasks - topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + calculated_fields: + # Topic name for Calculated Field (CF) events from Rule Engine + event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" + # Topic name for Calculated Field (CF) compacted states + state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" # Interval in milliseconds to poll messages by CF (Rule Engine) microservices - poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" # Enable/disable a separate consumer per partition for CF queue - consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + # Thread pool size for processing of the incoming messages + pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 073f47d59b..1b743316bd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,15 +644,15 @@ public class ProtoUtils { return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { - if (telemetryProto.hasAttrKv()) { - return fromProto(telemetryProto.getAttrKv().getValue()); - } else if (telemetryProto.hasTsKv()) { - return fromProto(telemetryProto.getTsKv()); - } else { - throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); - } - } +// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { +// if (telemetryProto.hasAttrKv()) { +// return fromProto(telemetryProto.getAttrKv().getValue()); +// } else if (telemetryProto.hasTsKv()) { +// return fromProto(telemetryProto.getTsKv()); +// } else { +// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); +// } +// } public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); @@ -673,12 +673,12 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { - return TransportProtos.AttributeKvProto.newBuilder() - .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) - .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) - .build(); - } +// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { +// return TransportProtos.AttributeKvProto.newBuilder() +// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) +// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) +// .build(); +// } public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4e719a02df..288c923aaa 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -819,6 +819,35 @@ message CalculatedFieldIdProto { int64 calculatedFieldIdLSB = 2; } +message SingleValueProto { + int64 ts = 1; + int64 version = 2; + KeyValueType type = 3; + bool has_v = 4; + bool bool_v = 5; + int64 long_v = 6; + double double_v = 7; + string string_v = 8; + string json_v = 9; +} + +message SingleValueArgumentProto { + string argName = 1; + SingleValueProto value = 2; +} + +message RollingArgumentProto { + string argName = 1; + repeated SingleValueProto values = 2; +} + +message CalculatedFieldStateProto { + CalculatedFieldEntityCtxIdProto id = 1; + // int32 version = 2; + repeated SingleValueArgumentProto singleValueArguments = 3; + repeated RollingArgumentProto rollingValueArguments = 4; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 53bdc78c93..7ac938f52f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -61,9 +61,11 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; - @Value("${queue.calculated-fields.topic}") - private String cfTopic; - @Value("${queue.calculated-fields.partitions:10}") + @Value("${queue.calculated_fields.event_topic}") + private String cfEventTopic; + @Value("${queue.calculated_fields.state_topic}") + private String cfStateTopic; + @Value("${queue.calculated_fields.partitions:10}") private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @@ -76,6 +78,8 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; + public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -116,9 +120,8 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - partitionSizesMap.put(cfKey, cfPartitions); - partitionTopicsMap.put(cfKey, cfTopic); + partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); + partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 927c311a2d..8dec36c15c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -35,6 +35,7 @@ public class TopicService { private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbCalculatedFieldNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentReferenceHashMap tbEdgeEventsNotificationTopics = new ConcurrentReferenceHashMap<>(); /** @@ -62,6 +63,11 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } + public TopicPartitionInfo getCalculatedFieldNotificationsTopic(String serviceId) { + return tbCalculatedFieldNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo("calculated_field", serviceId)); + } + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)); } @@ -71,7 +77,11 @@ public class TopicService { } private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + return buildNotificationsTopicPartitionInfo(serviceType.name().toLowerCase(), serviceId); + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(String serviceType, String serviceId) { + return buildTopicPartitionInfo(serviceType + ".notifications." + serviceId, null, null, false); } public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 3bb0c56f9a..57a4941981 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,10 +23,13 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -53,7 +56,7 @@ public class PartitionChangeEvent extends TbApplicationEvent { } public Set getCalculatedFieldsPartitions() { - return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index d70cad159b..c26e2d15c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -53,6 +54,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, @@ -62,6 +64,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, InMemoryStorage storage) { this.topicService = topicService; this.coreSettings = coreSettings; @@ -71,6 +74,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.storage = storage; } @@ -139,6 +143,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); @@ -209,6 +238,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index c4002f4d3e..0b3df5bccf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -159,4 +160,6 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index c406aeb311..76dad05393 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -109,11 +112,22 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory } /** - * Used to consume high priority messages by TB Core Service + * Used to consume high priority messages by TB Rule Engine Service * * @return */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + + TbQueueConsumer> createCalculatedFieldStateConsumer(); + + TbQueueProducer> createCalculatedFieldStateProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java new file mode 100644 index 0000000000..22bbd7e0f3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCalculatedFieldSettings { + + @Value("${queue.calculated_fields.event_topic}") + private String eventTopic; + + @Value("${queue.calculated_fields.state_topic}") + private String stateTopic; + + +} From 4c71b9d5f68907dc229fa181a3c3f43aafcf84b7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 13:56:44 +0200 Subject: [PATCH 082/281] WIP: Cluster mode implementation --- .../service/cf/CalculatedFieldCache.java | 4 +- .../cf/CalculatedFieldExecutionService.java | 12 +- .../cf/DefaultCalculatedFieldCache.java | 11 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++++- .../cf/ctx/state/CalculatedFieldCtx.java | 43 +++++++ ...faultTbCalculatedFieldConsumerService.java | 8 +- .../queue/DefaultTbClusterService.java | 12 +- .../TbCalculatedFieldConsumerService.java | 15 +++ .../AbstractSubscriptionService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 38 +++--- .../server/cluster/TbClusterService.java | 3 + .../queue/discovery/HashPartitionService.java | 3 +- .../queue/discovery/PartitionService.java | 2 + .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbCoreQueueProducerProvider.java | 16 +++ .../provider/TbQueueProducerProvider.java | 8 +- .../TbRuleEngineProducerProvider.java | 14 +++ .../provider/TbRuleEngineQueueFactory.java | 8 +- .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 11 ++ 20 files changed, 298 insertions(+), 54 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 8730aeeedf..ea55894432 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,9 +36,9 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e18c8b4119..2507546013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -22,13 +24,13 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { /** - * Push incoming telemetry to the CF processing queue for async processing. - * @param request - telemetry request; - * @param callback - callback to be executed when the message is ack by the queue. + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * @param request - telemetry save request; + * @param request - telemetry save result; */ - void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); +// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 868001d0d2..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbelInvokeService tbelInvokeService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -105,7 +107,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); @@ -128,9 +130,12 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } return getCalculatedFieldsByEntityId(entityId).stream() - .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> getCalculatedFieldCtx(cf.getId())) .toList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..0f9b948a9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,12 +33,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -60,6 +61,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; @@ -72,7 +74,9 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -103,6 +107,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -113,6 +119,16 @@ import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @RequiredArgsConstructor public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -121,7 +137,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; - private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -169,6 +184,70 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return "calculated-field-scheduled"; } + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ////TODO: IM to push to CF queue + return null; + } + + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) + .forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); + }); + } else { + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); + } + }); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -374,7 +453,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; @@ -391,7 +470,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { @@ -830,4 +909,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index cb4052b7df..1a9f999497 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -20,15 +20,18 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -41,6 +44,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final Map mainEntityArguments; + private final Map> linkedEntityArguments; + private final Map, String> referencedEntityKeys; private final List argNames; private Output output; @@ -55,6 +61,17 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.mainEntityArguments = new HashMap<>(); + this.linkedEntityArguments = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + var refId = entry.getValue().getRefEntityId(); + var refKey = entry.getValue().getRefEntityKey(); + if (refId == null) { + mainEntityArguments.put(refKey, entry.getKey()); + } else { + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + } + } this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), @@ -82,4 +99,30 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values) { + return matches(mainEntityArguments, values); + } + + public boolean linkMatches(EntityId entityId, List values) { + var map = linkedEntityArguments.get(entityId); + if (map == null) { + return false; + } else { + return matches(map, values); + } + } + + private static boolean matches(Map argMap, List values) { + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 551b0ddbc2..725e0fdd79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,7 +69,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -108,7 +109,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -336,6 +339,13 @@ public class DefaultTbClusterService implements TbClusterService { toTransportNfs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 0bb8f76398..21cb902ad6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -116,4 +116,12 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList }, executor); } + protected static Consumer safeCallback(FutureCallback callback) { + if (callback != null) { + return callback::onFailure; + } else { + return throwable -> {}; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index c9602b4650..71716a16e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.telemetry; -import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -53,7 +52,6 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -149,14 +147,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(resultFuture, request.getCallback()); + DonAsynchron.withCallback(resultFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request, result); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return resultFuture; } @@ -171,6 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); + //TODO: IM to push to CF queue addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @@ -268,27 +266,21 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes)); } private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice)); } private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts)); } private void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, List ts) { @@ -308,9 +300,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, updated, TbCallback.EMPTY); subscriptionManagerService.onTimeSeriesDelete(tenantId, entityId, deleted, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys); - }); + }, () -> TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys)); } private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 69334b774f..151d093c4f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -76,6 +77,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 7ac938f52f..5c6bf545b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -323,7 +323,8 @@ public class HashPartitionService implements PartitionService { } } - private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { + @Override + public TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b5744981bd..8e2152830e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -37,6 +37,8 @@ public interface PartitionService { TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId); + List resolveAll(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..69b3dff1f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -160,6 +161,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9cf18e6cb4..77f3cbcdfe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -48,6 +50,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -66,6 +70,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -124,4 +130,14 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeEvents; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + return toCalculatedFieldNotifications; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ec31763baa..2a151d58ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -76,7 +78,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return @@ -91,4 +93,8 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeEventsMsgProducer(); + TbQueueProducer> getCalculatedFieldsMsgProducer(); + + TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e9f7773a26..6a73dce1cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -47,6 +49,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; + private TbQueueProducer> toCalculatedFields; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -64,6 +67,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); } @Override @@ -121,4 +125,14 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 7960cbcf32..88b0f0b1ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -111,4 +112,13 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index cd4fa12df0..09c835e5fe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -107,4 +108,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + } From 2cc0d6f513bdd4f5873e62af1db51d5d6b10eab4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 23 Jan 2025 15:49:31 +0200 Subject: [PATCH 083/281] added implementations for consumer/producer methods --- ...faultTbCalculatedFieldConsumerService.java | 8 +- .../TbCalculatedFieldConsumerService.java | 15 ++++ .../src/main/resources/thingsboard.yml | 2 + .../queue/kafka/TbKafkaTopicConfigs.java | 5 ++ .../provider/KafkaMonolithQueueFactory.java | 81 +++++++++++++++++++ .../provider/KafkaTbCoreQueueFactory.java | 15 ++++ .../KafkaTbRuleEngineQueueFactory.java | 72 ++++++++++++++++- .../queue/provider/TbCoreQueueFactory.java | 8 +- .../provider/TbRuleEngineQueueFactory.java | 8 +- .../dao/sql/event/EventInsertRepository.java | 2 +- 10 files changed, 202 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 94c1bef71c..fdd9923292 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1619,6 +1619,8 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field topics + calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..cdd0add38b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,8 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.calculated-field:}") + private String calculatedFieldProperties; @Getter private Map coreConfigs; @@ -79,6 +81,8 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map calculatedFieldConfigs; @PostConstruct private void init() { @@ -97,6 +101,7 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..01e174023c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -54,6 +57,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -94,6 +99,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -106,6 +112,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; @@ -119,6 +126,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +141,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -490,6 +499,75 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId())); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -522,5 +600,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..aceccf7e58 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -53,6 +54,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -107,6 +110,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -119,6 +123,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -439,6 +444,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..46b35f9acd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,6 +23,9 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -49,6 +52,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -71,6 +75,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -81,6 +86,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -90,7 +96,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, + TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -101,6 +107,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -111,6 +118,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -293,6 +301,65 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-calculated-field-notifications-node-") + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -313,5 +380,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..673ac3a434 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 9307b9e9be..bdb675cbb7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -83,7 +83,7 @@ public class EventInsertRepository { " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + - " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + " (id, tenant_id, ts, entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } From 564162644360fddc6e37c7e29d485d72b88dc1e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 10:53:13 +0200 Subject: [PATCH 084/281] added processNotification impl --- .../cf/CalculatedFieldExecutionService.java | 14 +- ...efaultCalculatedFieldExecutionService.java | 309 ++++++++---------- .../cf/ctx/state/CalculatedFieldCtx.java | 29 +- ...faultTbCalculatedFieldConsumerService.java | 71 ++-- .../queue/DefaultTbClusterService.java | 36 +- .../queue/DefaultTbCoreConsumerService.java | 37 --- .../DefaultTelemetrySubscriptionService.java | 4 +- .../server/cluster/TbClusterService.java | 2 + .../server/common/util/ProtoUtils.java | 94 ------ .../provider/KafkaTbCoreQueueFactory.java | 16 + .../queue/provider/TbCoreQueueFactory.java | 4 +- 11 files changed, 245 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 2507546013..1ad0377e27 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { @@ -30,18 +32,18 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(AttributesSaveRequest request); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); - void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); +// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); + void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0f8cdd4fd4..cbd5445cdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,17 +33,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -74,7 +72,14 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -87,15 +92,12 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -112,7 +114,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -195,7 +196,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + @Override + public void pushRequestToQueue(AttributesSaveRequest request) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -223,31 +232,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ////TODO: IM to push to CF queue - return null; - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -318,19 +302,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { + ComponentLifecycleEvent event = proto.getEvent(); + if (ComponentLifecycleEvent.DELETED.equals(event)) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { + if (ComponentLifecycleEvent.UPDATED.equals(event)) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { @@ -340,14 +325,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (cf != null) { calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -492,30 +477,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(request); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } +// @Override +// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { +// try { +// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); +// +// if (proto.getLinksList().isEmpty()) { +// onTelemetryUpdate(request); +// return; +// } +// +// proto.getLinksList().forEach(ctxIdProto -> { +// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); +// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); +// +// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); +// if (!updatedTelemetry.isEmpty()) { +// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); +// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); +// } +// }); +// } catch (Exception e) { +// log.trace("Failed to process telemetry update msg: [{}]", proto, e); +// } +// } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -526,50 +511,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - @Override - public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); - } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(entityId, profileId, callback); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + } + if (proto.getAdded()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } + if (proto.getUpdated()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); } } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); + clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); } } catch (Exception e) { - log.trace("Failed to process profile entity msg: [{}]", proto, e); + log.trace("Failed to process entity update msg: [{}]", proto, e); } } @@ -582,7 +558,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() - .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId())) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -775,88 +751,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { - return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); - } - - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( - CalculatedFieldTelemetryUpdateRequest request, List links - ) { - TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); - - builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(request.getEntityId().getEntityType().name()) - .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); - - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - - for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { - builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); - } - - if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { - builder.setScope(attributeUpdateRequest.getScope().name()); - } - - for (KvEntry entry : request.getKvEntries()) { - TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); - if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { - telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); - } - if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { - telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); - } - builder.addUpdatedTelemetry(telemetryBuilder.build()); - } - - return builder.build(); - } - - private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - return attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build(); - } +// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { +// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() +// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) +// .setEntityType(ctxId.entityId().getEntityType().name()) +// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) +// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) +// .build(); +// } +// +// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { +// return TransportProtos.CalculatedFieldIdProto.newBuilder() +// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) +// .build(); +// } private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); @@ -905,6 +815,51 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + for (TsKvEntry entry : request.getEntries()) { + telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + for (AttributeKvEntry entry : request.getEntries()) { + telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + for (CalculatedFieldId cfId : calculatedFieldIds) { + CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); + calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); + calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); + telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + } + + return telemetryMsg; + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a9f999497..e4abbee4cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; @@ -99,20 +101,35 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values, AttributeScope scope) { + return matchesAttributes(mainEntityArguments, values, scope); + } + + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributes(map, values, scope); + } + public boolean matches(List values) { - return matches(mainEntityArguments, values); + return matchesTimeSeries(mainEntityArguments, values); } public boolean linkMatches(EntityId entityId, List values) { var map = linkedEntityArguments.get(entityId); - if (map == null) { - return false; - } else { - return matches(map, values); + return map != null && matchesTimeSeries(map, values); + } + + private static boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + for (AttributeKvEntry attrKv : values) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } } + return false; } - private static boolean matches(Map argMap, List values) { + private boolean matchesTimeSeries(Map argMap, List values) { for (TsKvEntry tsKv : values) { ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); if (argMap.containsKey(latestKey)) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ff4634f957..4eb3f0cd89 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -24,13 +25,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -157,8 +162,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { - ToCalculatedFieldNotificationMsg notification = msg.getValue(); - + ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); + if (toCfNotification.hasComponentLifecycle()) { + forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + } else if (toCfNotification.hasEntityUpdateMsg()) { + forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + } callback.onSuccess(); } @@ -184,41 +193,29 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // -// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { -// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); -// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { -// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 725e0fdd79..b510b813d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,8 +69,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -80,6 +79,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -109,7 +110,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @@ -346,6 +346,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); @@ -809,16 +816,23 @@ public class DefaultTbClusterService implements TbClusterService { } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - TransportProtos.CalculatedFieldMsgProto msg = builder.build(); - pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); + builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + TransportProtos.ComponentLifecycleEvent event; + if (added) { + event = TransportProtos.ComponentLifecycleEvent.CREATED; + } else if (updated) { + event = TransportProtos.ComponentLifecycleEvent.UPDATED; + } else { + event = TransportProtos.ComponentLifecycleEvent.DELETED; + } + builder.setEvent(event); + + pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c598540ff2..e3a1ca22d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -702,42 +701,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { - var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { - var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 71716a16e9..0f13dbb2f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -168,7 +168,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); - //TODO: IM to push to CF queue + DonAsynchron.withCallback(saveFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 151d093c4f..b28bf73c89 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 1b743316bd..3bace4d91f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -59,7 +58,6 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -630,98 +628,6 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromProto(TransportProtos.TsKvProto proto) { - TransportProtos.KeyValueProto kvProto = proto.getKv(); - String key = kvProto.getKey(); - KvEntry entry = switch (kvProto.getType()) { - case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); - case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); - case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); - case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); - case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); - default -> null; - }; - return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); - } - -// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { -// if (telemetryProto.hasAttrKv()) { -// return fromProto(telemetryProto.getAttrKv().getValue()); -// } else if (telemetryProto.hasTsKv()) { -// return fromProto(telemetryProto.getTsKv()); -// } else { -// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); -// } -// } - - public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { - TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); - builder.setAttributeKey(key); - switch (scope) { - case CLIENT_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); - break; - case SERVER_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); - break; - case SHARED_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); - break; - default: - throw new IllegalArgumentException("Unsupported attribute scope: " + scope); - } - return builder.build(); - } - -// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { -// return TransportProtos.AttributeKvProto.newBuilder() -// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) -// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) -// .build(); -// } - - public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { - TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); - builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); - switch (attributeKvEntry.getDataType()) { - case BOOLEAN: - builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) - .setHasV(true) - .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); - break; - case LONG: - builder.setType(TransportProtos.KeyValueType.LONG_V) - .setHasV(true) - .setLongV(attributeKvEntry.getLongValue().orElse(0L)); - break; - case DOUBLE: - builder.setType(TransportProtos.KeyValueType.DOUBLE_V) - .setHasV(true) - .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); - break; - case STRING: - builder.setType(TransportProtos.KeyValueType.STRING_V) - .setHasV(true) - .setStringV(attributeKvEntry.getStrValue().orElse("")); - break; - case JSON: - builder.setType(TransportProtos.KeyValueType.JSON_V) - .setHasV(true) - .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); - break; - default: - builder.setHasV(false); - throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); - } - if (attributeKvEntry.getKey() != null) { - builder.setKey(attributeKvEntry.getKey()); - } - if (attributeKvEntry.getVersion() != null) { - builder.setVersion(attributeKvEntry.getVersion()); - } - return builder.build(); - } - public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index aceccf7e58..329f1d7544 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -96,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -138,6 +140,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -444,6 +447,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @Override public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); @@ -483,5 +496,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 69b3dff1f0..1ece8249f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,7 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -161,7 +161,7 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } - TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldMsgProducer(); TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); From 85119d02475a5cbfb6688bc609414f2d3d5bf5e0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 11:37:32 +0200 Subject: [PATCH 085/281] WIP: cluster mode implementation --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 11 +++-- ...faultTbCalculatedFieldConsumerService.java | 6 +-- .../queue/DefaultTbClusterService.java | 42 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 3 +- .../src/main/resources/thingsboard.yml | 2 +- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 1ad0377e27..93a0ecc75f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -23,6 +23,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import java.util.List; + public interface CalculatedFieldExecutionService { /** @@ -32,7 +34,7 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushRequestToQueue(AttributesSaveRequest request); + void pushRequestToQueue(AttributesSaveRequest request, List result); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index cbd5445cdf..d46a7515ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -197,14 +197,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(AttributesSaveRequest request) { + public void pushRequestToQueue(AttributesSaveRequest request, List result) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -827,7 +829,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { + //TODO: IM Use result in both methods to update the versions of telemetry/attributes. ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 4eb3f0cd89..19d6e295b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -62,9 +62,9 @@ import java.util.UUID; @Slf4j public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { - @Value("${queue.calculated_fields.poll_interval}") + @Value("${queue.calculated_fields.poll_interval:25}") private long pollInterval; - @Value("${queue.calculated_fields.pack_processing_timeout}") + @Value("${queue.calculated_fields.pack_processing_timeout:60000}") private long packProcessingTimeout; @Value("${queue.calculated_fields.consumer_per_partition:true}") private boolean consumerPerPartition; @@ -102,7 +102,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_CORE)) + .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b510b813d6..3064277bf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -102,6 +102,7 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -570,7 +571,9 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -581,7 +584,6 @@ public class DefaultTbClusterService implements TbClusterService { || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.NOTIFICATION_RULE) - || entityType.equals(EntityType.CALCULATED_FIELD) ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); @@ -594,6 +596,14 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } + if (entityType.equals(EntityType.CALCULATED_FIELD)) { + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -641,11 +651,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - boolean deviceTypeChanged = !device.getType().equals(old.getType()); - if (deviceTypeChanged) { + boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + if (deviceProfileChanged) { handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } - if (deviceNameChanged || deviceTypeChanged) { + if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { @@ -802,37 +812,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { var created = oldCalculatedField == null; - broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); } @Override public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); - } - - private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); - builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - TransportProtos.ComponentLifecycleEvent event; - if (added) { - event = TransportProtos.ComponentLifecycleEvent.CREATED; - } else if (updated) { - event = TransportProtos.ComponentLifecycleEvent.UPDATED; - } else { - event = TransportProtos.ComponentLifecycleEvent.DELETED; - } - builder.setEvent(event); - - pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 0f13dbb2f9..7f889d431f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -167,9 +167,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public void saveAttributesInternal(AttributesSaveRequest request) { log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); - addMainCallback(saveFuture, request.getCallback()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request); + calculatedFieldExecutionService.pushRequestToQueue(request, result); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fdd9923292..d62aac5e6c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1750,7 +1750,7 @@ queue: # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:60000}" # Enable/disable a separate consumer per partition for CF queue consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages From bd34ed5011ac6d854bd06dcd92095662bc1a16b0 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:34:18 +0200 Subject: [PATCH 086/281] Implemented calculated fields table --- .../core/http/calculated-fields.service.ts | 78 +++++++++ .../calculated-fields-table-config.ts | 157 ++++++++++++++++++ .../calculated-fields-table.component.html | 1 + .../calculated-fields-table.component.ts | 104 ++++++++++++ .../entity/entities-table.component.ts | 8 +- .../home/components/home-components.module.ts | 9 +- .../entity/entity-table-component.models.ts | 4 +- .../pages/device/device-tabs.component.html | 4 + .../app/shared/models/entity-type.models.ts | 15 +- .../assets/locale/locale.constant-en_US.json | 9 + 10 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/core/http/calculated-fields.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts new file mode 100644 index 0000000000..92eae2c25a --- /dev/null +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { PageData } from '@shared/models/page/page-data'; + +@Injectable({ + providedIn: 'root' +}) +// [TODO]: [Calculated fields] - implement when BE ready +export class CalculatedFieldsService { + + fieldsMock = [ + { + name: 'Calculated Field 1', + type: 'Simple', + expression: '1 + 2', + id: { + id: '1', + } + }, + { + name: 'Calculated Field 2', + type: 'Script', + expression: '${power}', + id: { + id: '2', + } + } + ]; + + constructor( + private http: HttpClient + ) { } + + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(this.fieldsMock[0]); + // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + return of(this.fieldsMock[1]); + // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + } + + public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(true); + // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public getCalculatedFields(query: any, + config?: RequestConfig): Observable> { + return of({ + data: this.fieldsMock, + totalPages: 1, + totalElements: 2, + hasNext: false, + }); + // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts new file mode 100644 index 0000000000..f2e2a64b49 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -0,0 +1,157 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { TranslateService } from '@ngx-translate/core'; +import { Direction } from '@shared/models/page/sort-order'; +import { MatDialog } from '@angular/material/dialog'; +import { TimePageLink } from '@shared/models/page/page-link'; +import { Observable, of } from 'rxjs'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { DialogService } from '@core/services/dialog.service'; +import { MINUTE } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { EntityService } from '@core/http/entity.service'; +import { EntityDebugSettings } from '@shared/models/entity.models'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { catchError, switchMap } from 'rxjs/operators'; + +export class CalculatedFieldsTableConfig extends EntityTableConfig { + + readonly calculatedFieldsDebugPerTenantLimitsConfiguration = + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + public entityId: EntityId = null, + private store: Store, + private viewContainerRef: ViewContainerRef, + private overlay: Overlay, + private cd: ChangeDetectorRef, + private utilsService: UtilsService, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + ) { + super(); + this.tableTitle = this.translate.instant('calculated-fields.label'); + this.detailsPanelEnabled = false; + this.selectionEnabled = true; + this.searchEnabled = true; + this.addEnabled = true; + this.entitiesDeleteEnabled = true; + this.actionsColumnTitle = ''; + this.entityType = EntityType.CALCULATED_FIELDS; + this.entityTranslations = entityTypeTranslations.get(EntityType.CALCULATED_FIELDS); + + this.entitiesFetchFunction = pageLink => this.fetchCalculatedFields(pageLink); + + this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; + + this.columns.push( + new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push( + new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push( + new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + + this.cellActionDescriptors.push( + { + name: '', + nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + icon: 'mdi:bug', + isEnabled: () => true, + iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', + onAction: ($event, entity) => this.onOpenDebugConfig($event, entity), + }, + { + name: this.translate.instant('action.edit'), + icon: 'edit', + isEnabled: () => true, + // // [TODO]: [Calculated fields] - implement edit + onAction: (_, entity) => {} + } + ); + } + + fetchCalculatedFields(pageLink: TimePageLink): Observable> { + return this.calculatedFieldsService.getCalculatedFields(pageLink); + } + + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + const { renderer, viewContainerRef } = this.getTable(); + if ($event) { + $event.stopPropagation(); + } + const trigger = $event.target as Element; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const debugStrategyPopover = this.popoverService.displayPopover(trigger, renderer, + viewContainerRef, EntityDebugSettingsPanelComponent, 'bottom', true, null, + { + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + maxDebugModeDuration: this.maxDebugModeDuration, + entityLabel: this.translate.instant('debug-settings.integration'), + ...debugSettings + }, + {}, + {}, {}, true); + debugStrategyPopover.tbComponentRef.instance.popover = debugStrategyPopover; + debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((settings: EntityDebugSettings) => { + this.onDebugConfigChanged(id.id, settings); + debugStrategyPopover.hide(); + }); + } + } + + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { + const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); + + if (!isDebugActive) { + return debugSettings?.failuresEnabled ? this.translate.instant('debug-settings.failures') : this.translate.instant('common.disabled'); + } else { + return this.durationLeft.transform(debugSettings?.allEnabledUntil) + } + } + + private isDebugActive(allEnabledUntil: number): boolean { + return allEnabledUntil > new Date().getTime(); + } + + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { + this.calculatedFieldsService.getCalculatedField(id).pipe( + switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), + catchError(() => of(null)), + takeUntilDestroyed(this.destroyRef), + ).subscribe(() => this.updateData()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html new file mode 100644 index 0000000000..38aa1487af --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts new file mode 100644 index 0000000000..a5b1ab34c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + Input, + OnInit, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { EntityService } from '@core/http/entity.service'; +import { DialogService } from '@core/services/dialog.service'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; + +@Component({ + selector: 'tb-calculated-fields-table', + templateUrl: './calculated-fields-table.component.html', + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CalculatedFieldsTableComponent implements OnInit { + + @Input() entityId: EntityId; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + this.entitiesTable.updateData(); + } + } + } + + @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; + + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + + private activeValue = false; + private dirtyValue = false; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + private store: Store, + private overlay: Overlay, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + private utilsService: UtilsService) { + } + + ngOnInit() { + this.dirtyValue = !this.activeValue; + + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.entityService, + this.dialogService, + this.translate, + this.dialog, + this.entityId, + this.store, + this.viewContainerRef, + this.overlay, + this.cd, + this.utilsService, + this.durationLeft, + this.popoverService, + this.destroyRef + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 9bb5fe4d8a..3c723f8080 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -25,8 +25,10 @@ import { OnChanges, OnDestroy, OnInit, + Renderer2, SimpleChanges, - ViewChild + ViewChild, + ViewContainerRef, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -141,7 +143,9 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa private router: Router, private elementRef: ElementRef, private fb: FormBuilder, - private zone: NgZone) { + private zone: NgZone, + public viewContainerRef: ViewContainerRef, + public renderer: Renderer2) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 32e509e842..1a6b9c08e0 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,6 +183,8 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; @NgModule({ declarations: @@ -326,7 +328,8 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], imports: [ CommonModule, @@ -463,11 +466,13 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], providers: [ WidgetComponentService, CustomDialogService, + DurationLeftPipe, {provide: EMBED_DASHBOARD_DIALOG_TOKEN, useValue: EmbedDashboardDialogComponent}, {provide: COMPLEX_FILTER_PREDICATE_DIALOG_COMPONENT_TOKEN, useValue: ComplexFilterPredicateDialogComponent}, {provide: DASHBOARD_PAGE_COMPONENT_TOKEN, useValue: DashboardPageComponent}, diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts index b6f634195e..8c4f856609 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts @@ -20,7 +20,7 @@ import { SafeHtml } from '@angular/platform-browser'; import { PageLink } from '@shared/models/page/page-link'; import { Timewindow } from '@shared/models/time/time.models'; import { EntitiesDataSource } from '@home/models/datasource/entity-datasource'; -import { ElementRef, EventEmitter } from '@angular/core'; +import { ElementRef, EventEmitter, Renderer2, ViewContainerRef } from '@angular/core'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -64,6 +64,8 @@ export interface IEntitiesTableComponent { paginator: MatPaginator; sort: MatSort; route: ActivatedRoute; + viewContainerRef: ViewContainerRef; + renderer: Renderer2; addEnabled(): boolean; clearSelection(): void; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 5e30694719..357bb587cc 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index e1b59a9243..5b540a6c54 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -49,7 +49,8 @@ export enum EntityType { OAUTH2_CLIENT = 'OAUTH2_CLIENT', DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', - MOBILE_APP = 'MOBILE_APP' + MOBILE_APP = 'MOBILE_APP', + CALCULATED_FIELDS = 'CALCULATED_FIELDS', } export enum AliasEntityType { @@ -478,6 +479,18 @@ export const entityTypeTranslations = new MapAre you sure you want to leave this page?", @@ -1027,6 +1034,8 @@ "city-max-length": "Specified city should be less than 256" }, "common": { + "name": "Name", + "type": "Type", "username": "Username", "password": "Password", "enter-username": "Enter username", From a652b31d7f880b1c13734c26c5f331e971a524e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 15:41:00 +0200 Subject: [PATCH 087/281] implementation processing notification --- .../service/cf/CalculatedFieldCache.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 7 +- .../cf/DefaultCalculatedFieldCache.java | 32 ++++ ...efaultCalculatedFieldExecutionService.java | 164 +++++++++++------- .../entitiy/EntityStateSourcingListener.java | 21 +-- .../DefaultSystemDataLoaderService.java | 7 +- ...faultTbCalculatedFieldConsumerService.java | 2 +- .../queue/DefaultTbClusterService.java | 74 ++++---- .../queue/DefaultTbCoreConsumerService.java | 41 ----- .../processing/AbstractConsumerService.java | 4 + .../TbRuleEngineQueueConsumerManager.java | 14 +- .../DefaultTelemetrySubscriptionService.java | 2 - .../server/cluster/TbClusterService.java | 8 +- .../server/common/util/ProtoUtils.java | 15 ++ common/proto/src/main/proto/queue.proto | 4 - .../server/dao/service/EntityServiceTest.java | 5 +- 16 files changed, 211 insertions(+), 194 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ea55894432..1ee1d4d562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -42,6 +41,10 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void evictProfile(TenantId tenantId, EntityId entityId); + + void evictEntity(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..836af03e5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,8 +20,9 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import java.util.List; @@ -42,9 +43,9 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); -// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..4278e74845 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,11 +22,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -172,6 +177,33 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void evictProfile(TenantId tenantId, EntityId entityId) { + log.debug("[{}] evict entity profile from cache.", entityId); + profileEntities.remove(entityId); + } + + @Override + public void evictEntity(TenantId tenantId, EntityId entityId) { + calculatedFieldFetchLock.lock(); + try { + profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); + if (EntityType.ASSET.equals(entityId.getEntityType())) { + Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); + if (asset != null) { + profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } else { + Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); + if (device != null) { + profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..beb347d29f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,6 +36,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -72,14 +73,17 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -92,7 +96,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -334,7 +340,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -401,8 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { + public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { @@ -418,15 +425,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); } } } catch (Exception e) { @@ -479,30 +483,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } -// @Override -// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { -// try { -// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); -// -// if (proto.getLinksList().isEmpty()) { -// onTelemetryUpdate(request); -// return; -// } -// -// proto.getLinksList().forEach(ctxIdProto -> { -// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); -// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); -// -// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); -// if (!updatedTelemetry.isEmpty()) { -// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); -// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); -// } -// }); -// } catch (Exception e) { -// log.trace("Failed to process telemetry update msg: [{}]", proto, e); -// } -// } + @Override + public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(proto, callback); + return; + } + + proto.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); + + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } + } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -753,23 +757,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } -// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { -// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() -// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) -// .setEntityType(ctxId.entityId().getEntityType().name()) -// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) -// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) -// .build(); -// } -// -// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { -// return TransportProtos.CalculatedFieldIdProto.newBuilder() -// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) -// .build(); -// } - private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); @@ -821,22 +808,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); - for (TsKvEntry entry : request.getEntries()) { - telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { - //TODO: IM Use result in both methods to update the versions of telemetry/attributes. + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - for (AttributeKvEntry entry : request.getEntries()) { - telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -854,15 +847,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); for (CalculatedFieldId cfId : calculatedFieldIds) { - CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); - calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); - calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); - telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); } return telemetryMsg; } + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, scope, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index ab8246ccf5..4703ed1606 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -88,7 +87,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -123,9 +122,6 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); - } default -> { } } @@ -150,7 +146,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -191,10 +187,6 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - case CALCULATED_FIELD -> { - CalculatedField calculatedField = (CalculatedField) event.getEntity(); - tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); - } default -> { } } @@ -275,15 +267,6 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { - CalculatedField calculatedField = (CalculatedField) entity; - CalculatedField oldCalculatedField = null; - if (oldEntity instanceof CalculatedField) { - oldCalculatedField = (CalculatedField) oldEntity; - } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); - } - private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd8a59c392..5b0c790215 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; @@ -98,9 +99,9 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -308,7 +309,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { jwtSettingsService.saveJwtSettings(jwtSettings); } - List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE,0)).getData(); + List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE, 0)).getData(); if (CollectionUtils.isNotEmpty(mobiles)) { mobiles.stream() .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) @@ -571,7 +572,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private void save(DeviceId deviceId, String key, boolean value) { if (persistActivityToTelemetry) { - ListenableFuture saveFuture = tsService.save( + ListenableFuture saveFuture = tsService.save( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..3df2653df6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -171,7 +171,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer callback.onSuccess(); } -// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { + // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { // var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); // var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); // var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 3064277bf4..04d6a53401 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,12 +38,10 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -347,6 +345,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); @@ -409,7 +414,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -418,7 +423,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); + handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -653,13 +658,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -673,10 +678,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -809,52 +814,41 @@ public class DefaultTbClusterService implements TbClusterService { } } - @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { - var created = oldCalculatedField == null; - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); } - @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); } - private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); - - broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); } - private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { - TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); + private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(profileId.getEntityType().name()); - builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); - builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + if (oldProfileId != null) { + builder.setEntityProfileType(oldProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + } + if (newProfileId != null) { + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + } builder.setAdded(added); + builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.ProfileEntityMsgProto msg = builder.build(); + TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); + pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..e7bde845f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -88,7 +86,6 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -110,7 +107,6 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -153,14 +149,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; - private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -183,7 +177,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -319,12 +310,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); - if (added) { - entitiesByProfile.add(entityId); - } else { - entitiesByProfile.remove(entityId); - } - } - private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index dac35bfc5c..5b1e5d7d79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,12 +178,16 @@ public abstract class AbstractConsumerService onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index b28bf73c89..d2fbcddfdf 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); @@ -125,8 +127,4 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); - - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); - } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 3bace4d91f..6a40f13688 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -628,6 +629,20 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 288c923aaa..ede796b12b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1591,8 +1591,6 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; -// CalculatedFieldMsgProto calculatedFieldMsg = 53; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1612,8 +1610,6 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; -// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index a26f345fb7..8c93245837 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -1824,7 +1825,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -2430,7 +2431,7 @@ public class EntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); From c332e7373f507199552a814c424dd98b07dd4233 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 15:45:00 +0200 Subject: [PATCH 088/281] WIP: CalculatedFieldConsumer refactoring --- .../cf/CalculatedFieldExecutionService.java | 6 ++ ...efaultCalculatedFieldExecutionService.java | 13 +++ ...faultTbCalculatedFieldConsumerService.java | 85 ++++++++++++++++++- .../queue/DefaultTbCoreConsumerService.java | 13 +-- .../queue/DefaultTbEdgeConsumerService.java | 13 +-- .../service/queue/PendingMsgHolder.java | 24 ++++++ 6 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..893dba92a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,6 +20,8 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -36,6 +38,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + + void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..2638f22254 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -234,6 +235,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } + @Override + public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + + callback.onSuccess(); + } + + @Override + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + + callback.onSuccess(); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..b36b34dd94 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,8 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -32,12 +34,19 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -50,12 +59,20 @@ import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.DefaultTbCoreConsumerService.PendingMsgHolder; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -132,7 +149,46 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { - + List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); + ConcurrentMap> pendingMap = orderedMsgList.stream().collect( + Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); + Future packSubmitFuture = consumersExecutor.submit(() -> { + orderedMsgList.forEach((element) -> { + UUID id = element.getUuid(); + TbProtoQueueMsg msg = element.getMsg(); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCalculatedFieldMsg toCfMsg = msg.getValue(); + pendingMsgHolder.setMsg(toCfMsg); + if (toCfMsg.hasTelemetryMsg()) { + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); + } + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + consumer.commit(); } @Override @@ -193,6 +249,33 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // + + private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + + } + + private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..ca42298742 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -269,7 +269,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -278,7 +278,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); - pendingMsgHolder.setToCoreMsg(toCoreMsg); + pendingMsgHolder.setMsg(toCoreMsg); if (toCoreMsg.hasToSubscriptionMgrMsg()) { log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); @@ -335,8 +335,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); @@ -346,12 +345,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -145,7 +145,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); - pendingMsgHolder.setToEdgeMsg(toEdgeMsg); + pendingMsgHolder.setMsg(toEdgeMsg); if (toEdgeMsg.hasEdgeNotificationMsg()) { pushNotificationToEdge(toEdgeMsg.getEdgeNotificationMsg(), 0, packProcessingRetries, callback); } @@ -161,20 +161,13 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } - private static class PendingMsgHolder { - @Getter - @Setter - private volatile ToEdgeMsg toEdgeMsg; - } - @Override protected ServiceType getServiceType() { return ServiceType.TB_CORE; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java new file mode 100644 index 0000000000..8793e45da6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import lombok.Setter; + +public class PendingMsgHolder { + @Getter @Setter + private volatile T msg; +} From b169dfab2793ef1b0721f1b4b4ab7545823bdf81 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:46:48 +0200 Subject: [PATCH 089/281] added license headers --- .../calculated-fields-table.component.html | 17 ++++++++++++++ .../calculated-fields-table.component.scss | 22 +++++++++++++++++++ .../calculated-fields-table.component.ts | 1 + 3 files changed, 40 insertions(+) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html index 38aa1487af..d627e16ce9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -1 +1,18 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss new file mode 100644 index 0000000000..ea3f7d90b7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + tb-entities-table { + .mat-drawer-container { + background-color: white; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index a5b1ab34c1..5449bce5a7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -42,6 +42,7 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; @Component({ selector: 'tb-calculated-fields-table', templateUrl: './calculated-fields-table.component.html', + styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) export class CalculatedFieldsTableComponent implements OnInit { From be4ea19b91636d2e9c1e927a4776c954972ebc8c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:20:55 +0200 Subject: [PATCH 090/281] added calculated fields typing --- .../core/http/calculated-fields.service.ts | 21 ++++++--- .../calculated-fields-table-config.ts | 21 ++++----- .../pages/device/device-tabs.component.html | 2 +- .../shared/models/calculated-field.models.ts | 44 +++++++++++++++++++ .../app/shared/models/entity-type.models.ts | 8 ++-- .../shared/models/id/calculated-field-id.ts | 26 +++++++++++ .../assets/locale/locale.constant-en_US.json | 3 +- 7 files changed, 103 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/shared/models/calculated-field.models.ts create mode 100644 ui-ngx/src/app/shared/models/id/calculated-field-id.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 92eae2c25a..66e0833128 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,6 +19,7 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; +import { CalculatedField } from '@shared/models/calculated-field.models'; @Injectable({ providedIn: 'root' @@ -30,7 +31,11 @@ export class CalculatedFieldsService { { name: 'Calculated Field 1', type: 'Simple', - expression: '1 + 2', + configuration: { + expression: '1 + 2', + type: 'SIMPLE', + }, + entityId: '1', id: { id: '1', } @@ -38,23 +43,27 @@ export class CalculatedFieldsService { { name: 'Calculated Field 2', type: 'Script', - expression: '${power}', + entityId: '2', + configuration: { + expression: '${power}', + type: 'SIMPLE', + }, id: { id: '2', } } - ]; + ] as any[]; constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); } @@ -65,7 +74,7 @@ export class CalculatedFieldsService { } public getCalculatedFields(query: any, - config?: RequestConfig): Observable> { + config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, totalPages: 1, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f2e2a64b49..22f742b9d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,8 +39,9 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, switchMap } from 'rxjs/operators'; +import { CalculatedField } from '@shared/models/calculated-field.models'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; @@ -62,31 +63,31 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; this.columns.push( - new EntityTableColumn('name', 'common.name', '33%')); + new EntityTableColumn('name', 'common.name', '33%')); this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); + new EntityTableColumn('type', 'common.type', '50px')); this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); this.cellActionDescriptors.push( { name: '', - nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), icon: 'mdi:bug', isEnabled: () => true, iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', @@ -102,11 +103,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: TimePageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(pageLink); } - onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { const { renderer, viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 357bb587cc..ab59f412cd 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -33,7 +33,7 @@ + label="{{ 'entity.type-calculated-fields' | translate }}" #calculatedFieldsTab="matTab"> , HasVersion, HasTenantId { + entityId: string; + type: CalculatedFieldType; + name: string; + debugSettings?: EntityDebugSettings; + externalId?: string; + createdTime?: number; + configuration: CalculatedFieldConfiguration; +} + +export enum CalculatedFieldType { + SIMPLE = 'SIMPLE', + COMPLEX = 'COMPLEX', +} + +export interface CalculatedFieldConfiguration { + type: CalculatedFieldConfigType; + expression: string; + arguments: Record; +} + +export enum CalculatedFieldConfigType { + SIMPLE = 'SIMPLE', + SCRIPT = 'SCRIPT', +} diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index 5b540a6c54..ee39be0f27 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -50,7 +50,7 @@ export enum EntityType { DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', MOBILE_APP = 'MOBILE_APP', - CALCULATED_FIELDS = 'CALCULATED_FIELDS', + CALCULATED_FIELD = 'CALCULATED_FIELD', } export enum AliasEntityType { @@ -481,10 +481,10 @@ export const entityTypeTranslations = new Map Date: Fri, 24 Jan 2025 17:24:56 +0200 Subject: [PATCH 091/281] adjusted typing --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 66e0833128..72c4f6c12c 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -20,6 +20,7 @@ import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' @@ -73,7 +74,7 @@ export class CalculatedFieldsService { // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(query: any, + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, @@ -81,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From 41b0963884edc21bb28569701d66fd269e7daa9e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:34:25 +0200 Subject: [PATCH 092/281] updated endpoint --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 72c4f6c12c..5df4a84949 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -61,17 +61,17 @@ export class CalculatedFieldsService { public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); - // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(true); - // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public getCalculatedFields(pageLink: PageLink, @@ -82,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, + // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From ea7e6797edba6e113c581d5650870d7029d0af25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 18:11:18 +0200 Subject: [PATCH 093/281] Implemented set entityId --- .../calculated-fields-table.component.ts | 22 ++++++++++++++----- .../shared/models/calculated-field.models.ts | 5 +---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 5449bce5a7..f98d24bd52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -47,14 +47,23 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; }) export class CalculatedFieldsTableComponent implements OnInit { - @Input() entityId: EntityId; + @Input() + set entityId(entityId: EntityId) { + if (this.entityIdValue !== entityId) { + this.entityIdValue = entityId; + this.entitiesTable.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.hasInitialized = true; + } + } + } @Input() set active(active: boolean) { if (this.activeValue !== active) { this.activeValue = active; - if (this.activeValue && this.dirtyValue) { - this.dirtyValue = false; + if (this.activeValue && this.hasInitialized) { + this.hasInitialized = false; this.entitiesTable.updateData(); } } @@ -65,7 +74,8 @@ export class CalculatedFieldsTableComponent implements OnInit { calculatedFieldsTableConfig: CalculatedFieldsTableConfig; private activeValue = false; - private dirtyValue = false; + private hasInitialized = false; + private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, private entityService: EntityService, @@ -83,7 +93,7 @@ export class CalculatedFieldsTableComponent implements OnInit { } ngOnInit() { - this.dirtyValue = !this.activeValue; + this.hasInitialized = !this.activeValue; this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, @@ -91,7 +101,7 @@ export class CalculatedFieldsTableComponent implements OnInit { this.dialogService, this.translate, this.dialog, - this.entityId, + this.entityIdValue, this.store, this.viewContainerRef, this.overlay, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 714303e286..253bc58f39 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -17,13 +17,10 @@ import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/ent import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; -export interface CalculatedField extends BaseData, HasVersion, HasTenantId { - entityId: string; +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { type: CalculatedFieldType; - name: string; debugSettings?: EntityDebugSettings; externalId?: string; - createdTime?: number; configuration: CalculatedFieldConfiguration; } From 0e1cd69e34645ce1acd3401d85d45ef5f6fe43a2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 27 Jan 2025 15:55:40 +0200 Subject: [PATCH 094/281] updated cf consumer --- .../service/cf/CalculatedFieldCache.java | 4 - .../cf/CalculatedFieldExecutionService.java | 8 +- .../cf/DefaultCalculatedFieldCache.java | 32 ---- ...efaultCalculatedFieldExecutionService.java | 143 ++++++++++-------- .../entitiy/EntityStateSourcingListener.java | 22 ++- ...faultTbCalculatedFieldConsumerService.java | 63 ++++---- .../queue/DefaultTbClusterService.java | 47 +++++- .../processing/AbstractConsumerService.java | 4 - .../server/cluster/TbClusterService.java | 5 + .../server/common/util/ProtoUtils.java | 13 +- common/proto/src/main/proto/queue.proto | 2 + .../provider/TbCoreQueueProducerProvider.java | 3 +- 12 files changed, 188 insertions(+), 158 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 1ee1d4d562..ff3bda5da5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -41,10 +41,6 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); - void evictProfile(TenantId tenantId, EntityId entityId); - - void evictEntity(TenantId tenantId, EntityId entityId); - void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..7668acba4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; @@ -30,6 +31,7 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * * @param request - telemetry save request; * @param request - telemetry save result; */ @@ -37,6 +39,8 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); @@ -47,10 +51,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,16 +22,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -177,33 +172,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } - @Override - public void evictProfile(TenantId tenantId, EntityId entityId) { - log.debug("[{}] evict entity profile from cache.", entityId); - profileEntities.remove(entityId); - } - - @Override - public void evictEntity(TenantId tenantId, EntityId entityId) { - calculatedFieldFetchLock.lock(); - try { - profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); - if (EntityType.ASSET.equals(entityId.getEntityType())) { - Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); - if (asset != null) { - profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } else { - Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); - if (device != null) { - profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..2a4d42ddea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -66,7 +66,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -120,6 +119,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -242,14 +243,59 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(msg); + EntityId entityId = request.getEntityId(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + TenantId tenantId = request.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + + if (tpi.isMyPartition()) { + + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); - callback.onSuccess(); + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(request, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); + }); + } + } else { + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(msg).build(), null); + } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } } @Override public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(linkedMsg.getMsg()); + + if (linkedMsg.getLinksList().isEmpty()) { + onTelemetryMsg(linkedMsg.getMsg(), callback); + return; + } + + linkedMsg.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - callback.onSuccess(); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", linkedMsg, e); + } } @Override @@ -263,7 +309,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer resolvePartition = entityId -> { TopicPartitionInfo tpi; try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } @@ -378,6 +424,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { + EntityId entityId = calculatedField.getEntityId(); + ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); + } + case ASSET_PROFILE, DEVICE_PROFILE -> { + Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() + .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) + .collect(Collectors.toSet()); + tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); + } + default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() + + "' does not support calculated fields."); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; @@ -418,38 +484,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged; } - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { @@ -483,7 +517,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); + TopicPartitionInfo targetEntityTpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntity); if (targetEntityTpi.isMyPartition()) { Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { @@ -495,31 +529,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto, callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -534,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (tpi.isMyPartition()) { log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { @@ -824,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { long tsVersion = versions.get(i); - TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -858,8 +867,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 4703ed1606..95b340c362 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -87,7 +88,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -122,6 +123,9 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + } default -> { } } @@ -146,7 +150,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -187,6 +191,11 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + } default -> { } } @@ -267,6 +276,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..bd40784e8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -34,19 +32,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -170,6 +165,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasComponentLifecycleMsg()) { + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); + forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); + } else if (toCfMsg.hasEntityUpdateMsg()) { + log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); + forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,36 +220,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); } callback.onSuccess(); } - // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); -// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); -// } -// -// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { -// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); -// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); -// boolean added = profileEntityMsg.getAdded(); -// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); -// if (added) { -// entitiesByProfile.add(entityId); -// } else { -// entitiesByProfile.remove(entityId); -// } -// } -// - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); @@ -275,7 +253,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +265,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); @@ -299,6 +277,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } + private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { + var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); + if (entityUpdateMsg.getAdded()) { + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } else if (entityUpdateMsg.getDeleted()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + } else if (entityUpdateMsg.getUpdated()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } + } + private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); callback.onFailure(new RuntimeException("Message not handled!")); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 04d6a53401..b1a4b1859e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -68,6 +70,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -95,6 +98,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -145,6 +149,10 @@ public class DefaultTbClusterService implements TbClusterService { @Lazy private OtaPackageStateService otaPackageStateService; + @Autowired + @Lazy + private CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; private final TbAssetProfileCache assetProfileCache; @@ -342,21 +350,21 @@ public class DefaultTbClusterService implements TbClusterService { public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineMsgs.incrementAndGet(); } @Override public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + toRuleEngineMsgs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS } @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineNfs.incrementAndGet(); } @Override @@ -686,6 +694,20 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { + var created = oldCalculatedField == null; + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -827,7 +849,7 @@ public class DefaultTbClusterService implements TbClusterService { } private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); + CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -846,9 +868,22 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + + broadcastEntityUpdateEvent(msg); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + } - pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 5b1e5d7d79..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,16 +178,12 @@ public abstract class AbstractConsumerService> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFields; private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { From d3278f05bb4658bc109b71b927afd2b78e2bea88 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:34:35 +0200 Subject: [PATCH 095/281] WIP: Cluster mode refactoring --- .../server/actors/ActorSystemContext.java | 6 + .../server/actors/app/AppActor.java | 25 ++ .../CalculatedFieldEntityActor.java | 65 ++++ .../CalculatedFieldEntityActorCreator.java | 49 +++ ...CalculatedFieldEntityMessageProcessor.java | 199 +++++++++++ .../CalculatedFieldLinkedTelemetryMsg.java | 40 +++ .../CalculatedFieldManagerActor.java | 84 +++++ .../CalculatedFieldManagerActorCreator.java | 46 +++ ...alculatedFieldManagerMessageProcessor.java | 149 +++++++++ .../CalculatedFieldStateRestoreMsg.java | 40 +++ .../CalculatedFieldTelemetryMsg.java | 39 +++ .../EntityCalculatedFieldTelemetryMsg.java | 55 +++ .../calculatedField/MultipleTbCallback.java | 49 +++ .../actors/service/DefaultActorService.java | 11 + .../server/actors/tenant/TenantActor.java | 29 +- .../cf/CalculatedFieldExecutionService.java | 23 +- .../cf/CalculatedFieldInitService.java | 19 ++ .../cf/DefaultCalculatedFieldCache.java | 2 + ...efaultCalculatedFieldExecutionService.java | 313 ++++++++---------- .../cf/DefaultCalculatedFieldInitService.java | 64 ++++ .../cf/ctx/CalculatedFieldEntityCtxId.java | 3 +- .../cf/ctx/CalculatedFieldStateService.java | 9 +- .../ctx/state/BaseCalculatedFieldState.java | 5 + .../cf/ctx/state/CalculatedFieldCtx.java | 1 + .../cf/ctx/state/CalculatedFieldState.java | 1 + .../cf/ctx/state/RocksDBStateService.java | 14 +- .../ctx/state/SingleValueArgumentEntry.java | 16 + ...faultTbCalculatedFieldConsumerService.java | 48 +-- .../DefaultTelemetrySubscriptionService.java | 4 +- .../src/main/resources/thingsboard.yml | 2 + .../resources/application-test.properties | 2 + .../TbCalculatedFieldEntityActorId.java | 56 ++++ .../cf/configuration/ReferencedEntityKey.java | 2 + .../server/common/msg/MsgType.java | 11 +- .../msg/ToCalculatedFieldSystemMsg.java | 27 ++ .../common/msg/cf/CalculatedFieldInitMsg.java | 34 ++ .../msg/cf/CalculatedFieldLinkInitMsg.java | 34 ++ .../server/queue/util/AfterStartUp.java | 4 + .../resources/application-test.properties | 2 + 39 files changed, 1362 insertions(+), 220 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java create mode 100644 common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e6f541090c..09488bfe4e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -104,6 +104,7 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -507,6 +508,11 @@ public class ActorSystemContext { @Getter private EntityService entityService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 622812b327..9124dc0a9a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -111,6 +112,17 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } @@ -175,6 +187,19 @@ public class AppActor extends ContextAwareActor { } } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { + msg.getCallback().onSuccess(); + }); + } + + private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java new file mode 100644 index 0000000000..43f057ad01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; + +@Slf4j +public class CalculatedFieldEntityActor extends ContextAwareActor { + + private final CalculatedFieldEntityMessageProcessor processor; + + CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); + try { + processor.init(ctx); + log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); + } catch (Exception e) { + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); + throw new TbActorException("Failed to initialize device actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case CF_STATE_RESTORE_MSG: + processor.process((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java new file mode 100644 index 0000000000..c5f8ecf046 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.device.DeviceActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + private final EntityId entityId; + + public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { + super(context); + this.tenantId = tenantId; + this.entityId = entityId; + } + + @Override + public TbActorId createActorId() { + return new TbEntityActorId(entityId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldEntityActor(context, tenantId, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java new file mode 100644 index 0000000000..6bb6256563 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { + // (1 for result persistence + 1 for the state persistence ) + public static final int CALLBACKS_PER_CF = 2; + + final TenantId tenantId; + final EntityId entityId; + final CalculatedFieldExecutionService cfService; + + TbActorCtx ctx; + Map states = new HashMap<>(); + + CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.tenantId = tenantId; + this.entityId = entityId; + this.cfService = systemContext.getCalculatedFieldExecutionService(); + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) { + var proto = msg.getProto(); + var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); + List cfIdList = getCalculatedFieldIds(proto); + Set cfIdSet = new HashSet<>(cfIdList); + for (var ctx : msg.getEntityIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + for (var ctx : msg.getProfileIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + @SneakyThrows + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + } + + @SneakyThrows + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + } + + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + if (newArgValues.isEmpty()) { + callback.onSuccess(CALLBACKS_PER_CF); + } + CalculatedFieldState state = getOrInitState(ctx); + if (state.updateState(newArgValues)) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + + @SneakyThrows + private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = states.get(ctx.getCfId()); + if (state != null) { + return state; + } else { + ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } + return state; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { + List cfIds = new LinkedList<>(); + for (var cfId : proto.getPreviousCalculatedFieldsList()) { + cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); + } + return cfIds; + } + + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..0c352f5072 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldLinkedTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java new file mode 100644 index 0000000000..87292d3206 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public class CalculatedFieldManagerActor extends ContextAwareActor { + + private final CalculatedFieldManagerMessageProcessor processor; + + public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}] Starting CF manager actor.", processor.tenantId); + try { + processor.init(ctx); + log.debug("[{}] CF manager actor started.", processor.tenantId); + } catch (Exception e) { + log.warn("[{}] Unknown failure", processor.tenantId, e); + throw new TbActorException("Failed to initialize manager actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case PARTITION_CHANGE_MSG: + ctx.broadcastToChildren(msg, true); // TODO + break; + case CF_INIT_MSG: + processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); + break; + case CF_LINK_INIT_MSG: + processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + case CF_TELEMETRY_MSG: + processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + default: + return false; + } + return true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java new file mode 100644 index 0000000000..2a53089bed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.TbStringActorId; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + + public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { + super(context); + this.tenantId = tenantId; + } + + @Override + public TbActorId createActorId() { + return new TbStringActorId("CFM|" + tenantId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldManagerActor(context, tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java new file mode 100644 index 0000000000..4bc0589e95 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { + + private final Map calculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + + protected TbActorCtx ctx; + final TenantId tenantId; + + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.assetProfileCache = systemContext.getAssetProfileCache(); + this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.tenantId = tenantId; + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + var cf = msg.getCf(); + calculatedFields.put(cf.getId(), cf); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) + .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + msg.getCallback().onSuccess(); + } + + public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + var link = msg.getLink(); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + msg.getCallback().onSuccess(); + } + + public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { + if (calculatedFields.containsKey(msg.getId().cfId())) { + getOrCreateActor(msg.getId().entityId()).tell(msg); + } else { + // TODO: remove state from storage + } + } + + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + // process all cfs related to entity, or it's profile; + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + //TODO: Transfer only 'part' of the original callback. + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + // process all links (if any); + var links = getCalculatedFieldLinksByEntityId(entityId); + } + + private List getCalculatedFieldsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFields.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFieldLinks.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + protected TbActorRef getOrCreateActor(EntityId entityId) { + return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), + () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, + () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), + () -> true); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java new file mode 100644 index 0000000000..7c0e67f0ec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final CalculatedFieldEntityCtxId id; + private final CalculatedFieldState state; + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_RESTORE_MSG; + } + + @Override + public TenantId getTenantId() { + return id.tenantId(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..5cce83b00b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..8465bd8fcc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final List entityIdFields; + private final List profileIdFields; + private final TbCallback callback; + + public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, + List entityIdFields, + List profileIdFields, + TbCallback callback) { + this.tenantId = msg.getTenantId(); + this.entityId = msg.getEntityId(); + this.proto = msg.getProto(); + this.entityIdFields = entityIdFields; + this.profileIdFields = profileIdFields; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java new file mode 100644 index 0000000000..18e700f38c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbCallback implements TbCallback { + + private final AtomicInteger counter; + private final TbCallback callback; + + public MultipleTbCallback(int count, TbCallback callback) { + this.counter = new AtomicInteger(count); + this.callback = callback; + } + + @Override + public void onSuccess() { + if (counter.decrementAndGet() <= 0) { + callback.onSuccess(); + } + } + + public void onSuccess(int number) { + if (counter.addAndGet(-number) <= 0) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 492940a274..7e0d1b4447 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -49,6 +49,8 @@ public class DefaultActorService extends TbApplicationEventListener deletedDevices; + private TbActorRef cfActor; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -95,6 +98,11 @@ public class TenantActor extends RuleChainManagerActor { } else { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } + //TODO: IM - extend API usage to have CF Exec Enabled? Not in 4.0; + cfActor = ctx.getOrCreateChildActor(new TbStringActorId("CFM|" + tenantId), + () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, + () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), + () -> true); } catch (Exception e) { log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; @@ -159,12 +167,31 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } return true; } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + if (priority) { + cfActor.tellWithHighPriority(msg); + } else { + cfActor.tell(msg); + } + } + private boolean isMyPartition(EntityId entityId) { return systemContext.resolve(ServiceType.TB_CORE, tenantId, entityId).isMyPartition(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..d38c1c58ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,14 +15,22 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; @@ -30,16 +38,17 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * @param request - telemetry save request; - * @param request - telemetry save result; + * + * @param request - telemetry save request; + * @param callback */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); @@ -53,4 +62,6 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java new file mode 100644 index 0000000000..25f92e797f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +public interface CalculatedFieldInitService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..b0c9f6adde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..bb8ba8bb63 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -42,7 +41,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -63,7 +61,6 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -108,12 +105,13 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -167,7 +165,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -192,22 +189,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result) { + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, @@ -241,77 +238,84 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { - - callback.onSuccess(); + public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + Map> argFutures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return Futures.whenAllComplete(argFutures.values()).call(() -> { + var result = createStateByType(ctx.getCfType()); + result.updateState(argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + ))); + return result; + }, calculatedFieldCallbackExecutor); } @Override - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - - callback.onSuccess(); + public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(stateId, state, callback); } @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiTargetEntityMap = new HashMap<>(); - - for (CalculatedField cf : cfs) { - - Consumer resolvePartition = entityId -> { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); - } - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", - entityId, cf.getTenantId(), e.getMessage()); - } - }; - - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); - } else { - resolvePartition.accept(cfEntityId); - } - } - - for (var entry : tpiTargetEntityMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { - log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); - var future = calculatedFieldExecutor.submit(() -> { - try { - for (CalculatedFieldEntityCtxId ctxId : partition) { - restoreState(ctxId.cfId(), ctxId.entityId()); - } - } catch (Throwable t) { - log.error("Unexpected exception while restoring CalculatedField states", t); - throw t; - } - }); - result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); - } - } +// PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); +// Map> tpiTargetEntityMap = new HashMap<>(); +// +// for (CalculatedField cf : cfs) { +// +// Consumer resolvePartition = entityId -> { +// TopicPartitionInfo tpi; +// try { +// tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); +// if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { +// tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); +// } +// } catch (Exception e) { +// log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", +// entityId, cf.getTenantId(), e.getMessage()); +// } +// }; +// +// EntityId cfEntityId = cf.getEntityId(); +// if (isProfileEntity(cfEntityId)) { +// calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); +// } else { +// resolvePartition.accept(cfEntityId); +// } +// } +// +// for (var entry : tpiTargetEntityMap.entrySet()) { +// for (List partition : Lists.partition(entry.getValue(), 1000)) { +// log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); +// var future = calculatedFieldExecutor.submit(() -> { +// try { +// for (CalculatedFieldEntityCtxId ctxId : partition) { +// restoreState(ctxId.cfId(), ctxId.entityId()); +// } +// } catch (Throwable t) { +// log.error("Unexpected exception while restoring CalculatedField states", t); +// throw t; +// } +// }); +// result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); +// } +// } return result; } - private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - - if (restoredCtx != null) { - states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", calculatedFieldId); - } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); - } - } - @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { cleanupEntity(entityId); @@ -491,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); } } @@ -525,7 +529,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); +// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -569,9 +573,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - stateService.removeState(ctxId); } private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { @@ -601,7 +602,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); +// updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -613,96 +614,83 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - Map argumentsMap = new HashMap<>(argumentValues); - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - - CompletableFuture updateFuture = new CompletableFuture<>(); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - stateService.persistState(entityCtxId, calculatedFieldEntityCtx); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); - } - updateFuture.complete(null); - }; - - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); - - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } - - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } - - return calculatedFieldEntityCtx; - }); - } - - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); - } - } +// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { +// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); +// Map argumentsMap = new HashMap<>(argumentValues); +// +// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); +// +// states.compute(entityCtxId, (ctxId, ctx) -> { +// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); +// +// CompletableFuture updateFuture = new CompletableFuture<>(); +// +// Consumer performUpdateState = (state) -> { +// if (state.updateState(argumentsMap)) { +// calculatedFieldEntityCtx.setState(state); +// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); +// Map arguments = state.getArguments(); +// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && +// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); +// if (allArgsPresent) { +// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); +// } +// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); +// } +// updateFuture.complete(null); +// }; +// +// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); +// +// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); +// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() +// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); +// +// if (!allKeysPresent || requiresTsRollingUpdate) { +// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() +// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) +// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +// +// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) +// .addListener(() -> performUpdateState.accept(state), +// calculatedFieldCallbackExecutor); +// } else { +// performUpdateState.accept(state); +// } +// +// try { +// updateFuture.join(); +// } catch (Exception e) { +// log.trace("Failed to update state for ctxId [{}].", ctxId, e); +// throw new RuntimeException("Failed to update or initialize state.", e); +// } +// +// return calculatedFieldEntityCtx; +// }); +// } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); - } - - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { - throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); - } - List calculatedFieldIds = previousCalculatedFieldIds != null - ? new ArrayList<>(previousCalculatedFieldIds) - : new ArrayList<>(); - calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); - clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); - log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); } catch (Exception e) { - log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); } } @@ -781,15 +769,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); - - if (state == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return state; - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java new file mode 100644 index 0000000000..87dda7013f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +@Service +@TbRuleEngineComponent +@RequiredArgsConstructor +public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { + + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldStateService stateService; + private final ActorSystemContext actorSystemContext; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_INIT_SERVICE) + public void initCalculatedFieldDefinitions() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); + //TODO: combine with the DefaultCalculatedFieldCache. + + } + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 5fb90a3e46..6ce0a11ade 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; -public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { +public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 8bc5756f4e..2bf6b7e0f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + import java.util.Map; public interface CalculatedFieldStateService { - Map restoreStates(); + Map restoreStates(); - CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bc9a421e47..ca243e25b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -66,4 +66,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { return stateUpdated; } + @Override + public boolean isReady() { + //TODO: IM + return true; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e4abbee4cd..1a0a16254a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 3c4a680df1..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,4 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index db8950804c..aa82f2fc57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -36,24 +37,25 @@ public class RocksDBStateService implements CalculatedFieldStateService { private final RocksDBService rocksDBService; @Override - public Map restoreStates() { + public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) )); } @Override - public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) .orElse(null); } @Override - public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ + rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + callback.onSuccess(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d5080e90f..d5f7b1aee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,6 +21,10 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @Data @NoArgsConstructor @@ -34,6 +38,18 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private Long version; + public SingleValueArgumentEntry(TsKvProto entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + + public SingleValueArgumentEntry(AttributeValueProto entry) { + this.ts = entry.getLastUpdateTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..fb1d6b19ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -30,23 +28,20 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -166,10 +161,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer pendingMsgHolder.setMsg(toCfMsg); if (toCfMsg.hasTelemetryMsg()) { log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,9 +214,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); } callback.onSuccess(); } @@ -249,33 +244,20 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - var msg = linkedMsg.getMsg(); + private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); } - private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +269,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index df18640359..3f7f9c0b7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d62aac5e6c..30a3e51d8c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -441,6 +441,8 @@ actors: device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors edge_dispatcher_pool_size: "${ACTORS_SYSTEM_EDGE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for edge actors + cfm_dispatcher_pool_size: "${ACTORS_SYSTEM_CFM_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for CalculatedField manager actors + cfe_dispatcher_pool_size: "${ACTORS_SYSTEM_CFE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for CalculatedField entity actors tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 9951caa876..a1315cc8bd 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -35,6 +35,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java new file mode 100644 index 0000000000..66522cd08d --- /dev/null +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Objects; + +public class TbCalculatedFieldEntityActorId implements TbActorId { + + @Getter + private final EntityId entityId; + + public TbCalculatedFieldEntityActorId(EntityId entityId) { + this.entityId = entityId; + } + + @Override + public String toString() { + return entityId.getEntityType() + "|" + entityId.getId(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbCalculatedFieldEntityActorId that = (TbCalculatedFieldEntityActorId) o; + return entityId.equals(that.entityId); + } + + @Override + public int hashCode() { + // Magic number to ensure that the hash does not match with the hash of other actor id - (TbEntityActorId) + return 42 + Objects.hash(entityId); + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b49495d959..b4bcc77a17 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -27,4 +27,6 @@ public class ReferencedEntityKey { private ArgumentType type; private AttributeScope scope; + + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index b05bb75032..91419bee9c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -133,7 +133,16 @@ public enum MsgType { * Messages that are sent to and from edge session to start edge synchronization process */ EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, - EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG; + EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + + CF_INIT_MSG, // Sent to init particular calculated field; + CF_LINK_INIT_MSG, // Sent to init particular calculated field; + CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; + CF_TELEMETRY_MSG, + CF_ENTITY_TELEMETRY_MSG, + CF_LINKED_TELEMETRY_MSG, + CF_UPDATE_MSG, + CF_ENTITY_UPDATE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java new file mode 100644 index 0000000000..2da959ec7d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java new file mode 100644 index 0000000000..4d9f4ce414 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedField cf; + + @Override + public MsgType getMsgType() { + return MsgType.CF_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java new file mode 100644 index 0000000000..7f582e6ff8 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldLink link; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINK_INIT_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 53ff839c49..70ae853e1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,6 +38,10 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; + int CF_INIT_SERVICE = 10; + int CF_STATE_RESTORE_SERVICE = 11; + int CF_CONSUMER_SERVICE = 12; + int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; int AFTER_TRANSPORT_SERVICE = Integer.MAX_VALUE - 999; diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 1e11c77f9a..a1bb335ad0 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -145,6 +145,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 From 3d42a4ca04af2c21a89fc7fa11a8f9fd7adb970d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:57:39 +0200 Subject: [PATCH 096/281] Actor system implementation draft --- .../CalculatedFieldEntityMessageProcessor.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 6 ++++-- .../service/cf/ctx/state/CalculatedFieldState.java | 1 + .../org/thingsboard/server/common/util/ProtoUtils.java | 10 ++++++---- .../sql/cf/DefaultNativeCalculatedFieldRepository.java | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 6bb6256563..533eeb5e31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -78,6 +78,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.ctx = ctx; } + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -192,8 +196,4 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } - - public void process(CalculatedFieldStateRestoreMsg msg) { - states.put(msg.getId().cfId(), msg.getState()); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index bb8ba8bb63..3e37366da3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -837,8 +837,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if(calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..f261e586a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,5 +44,6 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 6a40f13688..9bbce7c522 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,11 +644,13 @@ public class ProtoUtils { } public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { - return TransportProtos.TsKvProto.newBuilder() + var builder = TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) - .setKv(toKeyValueProto(tsKvEntry)) - .setVersion(tsKvEntry.getVersion()) - .build(); + .setKv(toKeyValueProto(tsKvEntry)); + if (tsKvEntry.getVersion() != null) { + builder.setVersion(tsKvEntry.getVersion()); + } + return builder.build(); } public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index bb88982e3d..677234dc20 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -77,7 +77,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); - long version = (long) row.get("version"); + long version = row.get("version") != null ? (long) row.get("version") : 0; Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); From d54cb300d42757b929bbe94936e6d90b1f5db874 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 28 Jan 2025 08:43:29 +0200 Subject: [PATCH 097/281] added new endpoint --- .../controller/CalculatedFieldController.java | 30 +++++++++++++++++++ .../controller/ControllerConstants.java | 1 + 2 files changed, 31 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 4bf243034b..efe25e3883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -24,17 +25,26 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -81,6 +91,26 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } + @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", + notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @ResponseBody + public PageData getCalculatedFields( + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) + @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index c7a59cfd83..29625941bd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -96,6 +96,7 @@ public class ControllerConstants { protected static final String EDGE_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the edge name."; protected static final String EVENT_TEXT_SEARCH_DESCRIPTION = "The value is not used in searching."; protected static final String AUDIT_LOG_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on one of the next properties: entityType, entityName, userName, actionType, actionStatus."; + protected static final String CF_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the calculated field name."; protected static final String SORT_PROPERTY_DESCRIPTION = "Property of entity to sort by"; protected static final String SORT_ORDER_DESCRIPTION = "Sort order. ASC (ASCENDING) or DESC (DESCENDING)"; From 5e16db275cb628bf57d3afea6c2df78bcba86fd3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 Jan 2025 14:44:28 +0200 Subject: [PATCH 098/281] state and argument refactoring --- ...efaultCalculatedFieldExecutionService.java | 11 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 38 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 5 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/ScriptCalculatedFieldState.java | 9 +- .../ctx/state/SimpleCalculatedFieldState.java | 26 +- .../ctx/state/SingleValueArgumentEntry.java | 26 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 48 ++-- .../state/ScriptCalculatedFieldStateTest.java | 238 ++++++++++++++++++ .../state/SimpleCalculatedFieldStateTest.java | 213 ++++++++++++++++ .../state/SingleValueArgumentEntryTest.java | 71 ++++++ .../ctx/state/TsRollingArgumentEntryTest.java | 93 +++++++ 13 files changed, 725 insertions(+), 56 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..6f19090b6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; @@ -248,7 +247,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas argFutures.put(entry.getKey(), argValueFuture); } return Futures.whenAllComplete(argFutures.values()).call(() -> { - var result = createStateByType(ctx.getCfType()); + var result = createStateByType(ctx); result.updateState(argFutures.entrySet().stream() .collect(Collectors.toMap( Entry::getKey, // Keep the key as is @@ -798,10 +797,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return payload; } - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); + private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + return switch (ctx.getCfType()) { + case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); + case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index b261840bfd..be471a8ad9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -39,7 +39,7 @@ public interface ArgumentEntry { Object getValue(); - boolean hasUpdatedValue(ArgumentEntry entry); + boolean updateEntry(ArgumentEntry entry); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ca243e25b5..d623043cee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -16,14 +16,21 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; +import java.util.List; import java.util.Map; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + protected List requiredArguments; protected Map arguments; public BaseCalculatedFieldState() { - arguments = new HashMap<>(); + this.arguments = new HashMap<>(); + } + + public BaseCalculatedFieldState(List requiredArguments) { + this.requiredArguments = requiredArguments; + this.arguments = new HashMap<>(); } @Override @@ -44,22 +51,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { - if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { - existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); - } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { - existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { -// Long existingVersion = existingSingleValueEntry.getVersion(); -// Long newVersion = singleValueEntry.getVersion(); -// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { -// arguments.put(key, newEntry.copy()); -// } - arguments.put(key, newEntry.copy()); - } else { - arguments.put(key, newEntry.copy()); - } + if (existingEntry == null) { + validateNewEntry(newEntry); + arguments.put(key, newEntry.copy()); stateUpdated = true; + } else { + stateUpdated = existingEntry.updateEntry(newEntry); } } @@ -68,7 +65,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public boolean isReady() { - //TODO: IM - return true; + return arguments.keySet().containsAll(requiredArguments) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && + !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } + + protected void validateNewEntry(ArgumentEntry newEntry) { + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..b9834f18d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import net.objecthunter.exp4j.Expression; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -31,7 +32,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; @@ -56,6 +56,7 @@ public class CalculatedFieldCtx { private String expression; private TbelInvokeService tbelInvokeService; private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + private ThreadLocal customExpression; public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { this.cfId = calculatedField.getId(); @@ -86,6 +87,8 @@ public class CalculatedFieldCtx { this.tbelInvokeService = tbelInvokeService; if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } else { + this.customExpression = new ThreadLocal<>(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f261e586a4..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,6 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); - @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0421055fef..290fd95370 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -32,6 +33,10 @@ import java.util.TreeMap; @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + public ScriptCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -40,9 +45,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e16d310b3e..59b3009bb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -24,21 +24,32 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; +import java.util.List; import java.util.Map; @Data public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + public SimpleCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); + } + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); + ThreadLocal customExpression = ctx.getCustomExpression(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) @@ -47,9 +58,14 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build(); customExpression.set(expr); } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); + + for (Map.Entry entry : this.arguments.entrySet()) { + try { + expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index d5f7b1aee6..5117fcab0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,7 +21,6 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -35,7 +34,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private Long version; public SingleValueArgumentEntry(TsKvProto entry) { @@ -79,14 +77,28 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } - @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return this.ts != ((SingleValueArgumentEntry) entry).getTs(); - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); } + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() == this.ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.value = singleValueEntry.getValue(); + this.version = newVersion; + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1118e3af13..b86a51ca03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -62,31 +62,49 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return entry instanceof SingleValueArgumentEntry ? - !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : - !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); } @Override - public ArgumentEntry copy() { - return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { + return updateTsRollingEntry(tsRollingEntry); + } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + return updateSingleValueEntry(singleValueEntry); + } else { + throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); + } + } + + private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { + boolean updated = false; + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); + } + return updated; } - public void addTsRecord(Long key, Object value) { + private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + } + + private boolean addTsRecordIfAbsent(Long ts, Object value) { + if (!tsRecords.containsKey(ts)) { + addTsRecord(ts, value); + return true; + } + return false; + } + + private void addTsRecord(Long ts, Object value) { if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(key, value); + tsRecords.put(ts, value); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } } else { - log.warn("Argument type 'TS_ROLLING' only supports numeric values."); - } - } - - public void addAllTsRecords(Map newRecords) { - for (Map.Entry entry : newRecords.entrySet()) { - addTsRecord(entry.getKey(), entry.getValue()); + throw new IllegalArgumentException("Argument type " + getType() + " only supports numeric values."); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java new file mode 100644 index 0000000000..cbaea6575c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = DefaultTbelInvokeService.class) +public class ScriptCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); + + private final long ts = System.currentTimeMillis(); + + private ScriptCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + state = new ScriptCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); + + Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", assetHumidityArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + Map newArgs = Map.of("assetHumidity", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", newArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4);// will not be used for calculation + values.put(ts - 45000, 2);// will not be used for calculation + values.put(ts - 20, 0); + + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000);// will not be used + values.put(ts - 18, 0); + values.put(ts - 16, 0); + values.put(ts - 14, 0); + values.put(ts - 12, 0); + values.put(ts - 10, 0); + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isFalse(); + } + + private TsRollingArgumentEntry createRollingArgEntry() { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + long ts = System.currentTimeMillis(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + argumentEntry.setTsRecords(values); + return argumentEntry; + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(ASSET_ID); + calculatedField.setType(CalculatedFieldType.SCRIPT); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(DEVICE_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); + argument1.setRefEntityKey(refEntityKey1); + argument1.setLimit(5); + argument1.setTimeWindow(30000); + + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); + + config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java new file mode 100644 index 0000000000..58a981824c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -0,0 +1,213 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SimpleCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + + private SimpleCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), null); + state = new SimpleCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", key3ArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + Map newArgs = Map.of("key1", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); + } + + @Test + void testUpdateStateWhenRollingEntryPassed() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("output", 49.0)); + } + + @Test + void testPerformCalculationWhenPassedNotNumber() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key3", key3ArgEntry + )); + + assertThatThrownBy(() -> state.performCalculation(ctx)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument 'key2' is not a number."); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + + assertThat(state.isReady()).isFalse(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); + argument2.setRefEntityKey(refEntityKey2); + + Argument argument3 = new Argument(); + argument3.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); + argument3.setRefEntityKey(refEntityKey3); + + config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); + + config.setExpression("key1 + key2 + key3"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java new file mode 100644 index 0000000000..285da0b423 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SingleValueArgumentEntryTest { + + private SingleValueArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new SingleValueArgumentEntry(ts, 11, 363L); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWithThaSameTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.getVersion()).isEqualTo(369L); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java new file mode 100644 index 0000000000..b08c5f2a58 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TsRollingArgumentEntryTest { + + private TsRollingArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + entry = new TsRollingArgumentEntry(values); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenSingleValueEntryPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(4); + assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isFalse(); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 16); + values.put(ts - 10, 7); + values.put(ts - 5, 1); + newEntry.setTsRecords(values); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(5); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 40, 10, + ts - 30, 12, + ts - 20, 17, + ts - 10, 7, + ts - 5, 1 + )); + } + + @Test + void testUpdateEntryWhenValueIsNotNumber() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + + assertThatThrownBy(() -> entry.updateEntry(newEntry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + } + +} \ No newline at end of file From 5aabbd0f1ee683d99dffcde7325f0eb7253dbd05 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Jan 2025 15:37:36 +0200 Subject: [PATCH 099/281] Link dispatch implementation --- .../CalculatedFieldEntityActor.java | 5 +- ...CalculatedFieldEntityMessageProcessor.java | 59 ++++++- .../CalculatedFieldManagerActor.java | 2 + ...alculatedFieldManagerMessageProcessor.java | 125 +++++++++++++- ...tityCalculatedFieldLinkedTelemetryMsg.java | 42 +++++ .../calculatedField/MultipleTbCallback.java | 15 +- .../cf/CalculatedFieldExecutionService.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 159 ++++++++++-------- .../cf/ctx/state/CalculatedFieldCtx.java | 11 ++ ...faultTbCalculatedFieldConsumerService.java | 7 +- .../queue/DefaultTbClusterService.java | 13 ++ .../server/service/queue/TbPackCallback.java | 3 + .../server/cluster/TbClusterService.java | 3 + .../server/common/msg/queue/TbCallback.java | 8 + common/proto/src/main/proto/queue.proto | 3 +- 15 files changed, 365 insertions(+), 96 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index 43f057ad01..a5461c6152 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -43,7 +43,7 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); } catch (Exception e) { log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); - throw new TbActorException("Failed to initialize device actor", e); + throw new TbActorException("Failed to initialize CF entity actor", e); } } @@ -56,6 +56,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; + case CF_LINKED_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); + break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 533eeb5e31..cea3cce791 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,15 +18,19 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -40,6 +44,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -96,7 +102,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + var proto = msg.getProto(); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) { if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { @@ -120,8 +144,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); } + @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + Map newArgValues) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } @@ -159,8 +184,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + return mapToArguments(ctx.getMainEntityArguments(), data); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, data); + } + + private static Map mapToArguments(Map argNames, List data) { + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = argNames.get(key); @@ -177,8 +216,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, scope, attrDataList); + } + + private static Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -196,4 +246,5 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 87292d3206..1c198c660d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -73,6 +73,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); break; case CF_LINKED_TELEMETRY_MSG: + processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); + break; case CF_ENTITY_UPDATE_MSG: // processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bc0589e95..4bce7cb322 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,29 +16,52 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -50,18 +73,22 @@ import java.util.concurrent.CopyOnWriteArrayList; @Slf4j public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { - private final Map calculatedFields = new HashMap<>(); + private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + private final CalculatedFieldExecutionService cfService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; protected TbActorCtx ctx; final TenantId tenantId; + private final static int initFetchPackSize = 1024; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); + this.cfService = systemContext.getCalculatedFieldExecutionService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -73,11 +100,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - calculatedFields.put(cf.getId(), cf); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) - .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -99,14 +126,70 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - var proto = msg.getProto(); + // 2 = 1 for CF processing + 1 for links processing + MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); - //TODO: Transfer only 'part' of the original callback. - getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); + } else { + callback.onSuccess(); + } // process all links (if any); - var links = getCalculatedFieldLinksByEntityId(entityId); + List linkedCalculatedFields = filterCalculatedFieldLinks(msg); + var linksSize = linkedCalculatedFields.size(); + if (linksSize > 0) { + cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { + EntityId sourceEntityId = msg.getEntityId(); + var proto = msg.getProto(); + var linksList = proto.getLinksList(); + for (var linkProto : linksList) { + var link = toCalculatedFieldEntityCtxId(linkProto); + var targetEntityId = link.entityId(); + var targetEntityType = targetEntityId.getEntityType(); + var cf = calculatedFields.get(link.cfId()); + if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { + // iterate over all entities that belong to profile and push the message for corresponding CF + var entityIds = getEntitiesByProfile(targetEntityId); + if (!entityIds.isEmpty()) { + MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + } else { + msg.getCallback().onSuccess(); + } + } else { + // push the message to specific entity; + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); + getOrCreateActor(targetEntityId).tell(newMsg); + } + } + } + + private CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId(CalculatedFieldEntityCtxIdProto ctxIdProto) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var link : getCalculatedFieldLinksByEntityId(entityId)) { + CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); + if (ctx.linkMatches(entityId, proto)) { + result.add(ctx.toCalculatedFieldEntityCtxId()); + } + } + return result; } private List getCalculatedFieldsByEntityId(EntityId entityId) { @@ -131,6 +214,30 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private Set getEntitiesByProfile(EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getAssetService().findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getDeviceService().findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); @@ -139,7 +246,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware }; } - protected TbActorRef getOrCreateActor(EntityId entityId) { + private TbActorRef getOrCreateActor(EntityId entityId) { return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..47b91cbe65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java index 18e700f38c..312cf72bed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -15,35 +15,42 @@ */ package org.thingsboard.server.actors.calculatedField; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +@Slf4j public class MultipleTbCallback implements TbCallback { - + @Getter + private final UUID id; private final AtomicInteger counter; private final TbCallback callback; public MultipleTbCallback(int count, TbCallback callback) { + id = UUID.randomUUID(); this.counter = new AtomicInteger(count); this.callback = callback; } @Override public void onSuccess() { - if (counter.decrementAndGet() <= 0) { - callback.onSuccess(); - } + onSuccess(1); } public void onSuccess(int number) { + log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); if (counter.addAndGet(-number) <= 0) { + log.trace("[{}][{}] Done.", id, callback.getId()); callback.onSuccess(); } } @Override public void onFailure(Throwable t) { + log.warn("[{}][{}] onFailure.", id, callback.getId()); callback.onFailure(t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 806c224608..d8daae2e64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -53,6 +54,10 @@ public interface CalculatedFieldExecutionService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + + void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ @@ -65,6 +70,5 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..19561b73b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -34,6 +34,8 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -83,6 +85,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -636,60 +639,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } -// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { -// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); -// Map argumentsMap = new HashMap<>(argumentValues); -// -// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); -// -// states.compute(entityCtxId, (ctxId, ctx) -> { -// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); -// -// CompletableFuture updateFuture = new CompletableFuture<>(); -// -// Consumer performUpdateState = (state) -> { -// if (state.updateState(argumentsMap)) { -// calculatedFieldEntityCtx.setState(state); -// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); -// Map arguments = state.getArguments(); -// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && -// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); -// if (allArgsPresent) { -// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); -// } -// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); -// } -// updateFuture.complete(null); -// }; -// -// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); -// -// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); -// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() -// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); -// -// if (!allKeysPresent || requiresTsRollingUpdate) { -// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() -// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) -// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); -// -// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) -// .addListener(() -> performUpdateState.accept(state), -// calculatedFieldCallbackExecutor); -// } else { -// performUpdateState.accept(state); -// } -// -// try { -// updateFuture.join(); -// } catch (Exception e) { -// log.trace("Failed to update state for ctxId [{}].", ctxId, e); -// throw new RuntimeException("Failed to update or initialize state.", e); -// } -// -// return calculatedFieldEntityCtx; -// }); -// } @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { @@ -713,9 +662,74 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } catch (Exception e) { log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + callback.onFailure(e); } } + @Override + public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback) { + Map> unicasts = new HashMap<>(); + List broadcasts = new ArrayList<>(); + for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { + var linkEntityId = link.entityId(); + var linkEntityType = linkEntityId.getEntityType(); + // Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P. + boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); + if (broadcast) { + broadcasts.add(link); + } else { + TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); + } + } + MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); + if (!broadcasts.isEmpty()) { + broadcast(broadcasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + if (!unicasts.isEmpty()) { + unicast(unicasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + } + + private void unicast(Map> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); + unicasts.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), + ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + }); + } + + private void broadcast(List broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(mainCallback); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); + clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + } + + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + //TODO: IM: move to utils; + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -868,25 +882,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); - builder.setMsg(telemetryProto); - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - return builder.build(); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -948,4 +943,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private static class TbCallbackWrapper implements TbQueueCallback { + private final TbCallback callback; + + public TbCallbackWrapper(TbCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..37e402fbc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; import java.util.HashMap; @@ -143,4 +145,13 @@ public class CalculatedFieldCtx { } return false; } + + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { + //TODO: IM - implement + return true; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0ccecbbeca..0fcc04bd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -223,15 +223,16 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - // from upstream (maybe removed since we dont need to init state for each partition) + // from upstream (maybe removed since we don't need to init state for each partition) forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we dont need to update state for each partition) + // from upstream (maybe removed since we don't need to update state for each partition) forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + } else if (toCfNotification.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } - callback.onSuccess(); } private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b1a4b1859e..2783e826fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -194,6 +194,19 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg toCfMsg, TbQueueCallback callback) { + UUID msgId = UUID.randomUUID(); + TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + for (String serviceId : tbReServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + callback.onSuccess(null); // TODO: refactor to be fair, similar to multi-value callback; + } + @Override public void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, TenantId.SYS_TENANT_ID, tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index f9d15353c7..3402a357d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -23,9 +24,11 @@ import java.util.UUID; @Slf4j public class TbPackCallback implements TbCallback { private final TbPackProcessingContext ctx; + @Getter private final UUID id; public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + log.trace("[{}] CALLBACK CREATED", id); this.id = id; this.ctx = ctx; } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 269db0a73a..588b135891 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -62,6 +63,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastToCore(ToCoreNotificationMsg msg); + void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg build, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index a2d264a4ff..f8204fb0ae 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,10 @@ */ package org.thingsboard.server.common.msg.queue; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + public interface TbCallback { TbCallback EMPTY = new TbCallback() { @@ -30,6 +34,10 @@ public interface TbCallback { } }; + default UUID getId(){ + return EntityId.NULL_UUID; + } + void onSuccess(); void onFailure(Throwable t); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9e3e842cd9..69b912120e 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -803,7 +803,7 @@ message CalculatedFieldTelemetryMsgProto { message CalculatedFieldLinkedTelemetryMsgProto { CalculatedFieldTelemetryMsgProto msg = 1; - repeated CalculatedFieldEntityCtxIdProto links = 2; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1639,6 +1639,7 @@ message ToCalculatedFieldMsg { message ToCalculatedFieldNotificationMsg { ComponentLifecycleMsgProto componentLifecycle = 1; CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From a4aa2444acf62026d11f4601076885ba2ccc213c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 30 Jan 2025 11:01:45 +0200 Subject: [PATCH 100/281] added new endpoint --- .../controller/CalculatedFieldController.java | 35 ++++++++++--------- .../cf/DefaultTbCalculatedFieldService.java | 9 +++++ .../entitiy/cf/TbCalculatedFieldService.java | 5 +++ .../server/dao/cf/CalculatedFieldService.java | 2 ++ .../dao/cf/BaseCalculatedFieldService.java | 8 +++++ .../server/dao/cf/CalculatedFieldDao.java | 2 ++ .../dao/sql/cf/CalculatedFieldRepository.java | 4 +++ ...efaultNativeCalculatedFieldRepository.java | 2 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++++ 9 files changed, 56 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index efe25e3883..124dd8d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -32,6 +32,8 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; @@ -40,7 +42,8 @@ import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; @@ -91,25 +94,25 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", - notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody - public PageData getCalculatedFields( - @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) - @RequestParam int pageSize, - @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page, - @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) - @RequestParam(required = false) String textSearch, - @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) - @RequestParam(required = false) String sortProperty, - @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) - @RequestParam(required = false) String sortOrder) throws ThingsboardException { + public PageData getCalculatedFieldsByEntityId( + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); - return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + checkParameter("entityId", entityIdStr); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); + checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); + return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2e6e975636..c8c589691b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -74,6 +76,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); } + @Override + public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { + TenantId tenantId = user.getTenantId(); + checkEntityExistence(tenantId, entityId); + return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + } + @Override @Transactional public void delete(CalculatedField calculatedField, SecurityUser user) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 89931b8541..cb7e809059 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,6 +18,9 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { @@ -26,6 +29,8 @@ public interface TbCalculatedFieldService { CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + void delete(CalculatedField calculatedField, SecurityUser user); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 3a508a5c08..f08d54190f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -44,6 +44,8 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findAllCalculatedFields(PageLink pageLink); + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 26ed4134cc..d37a4a53c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -118,6 +118,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findAll(pageLink); } + @Override + public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { validateId(tenantId, id -> INCORRECT_TENANT_ID + id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 39663d0afc..23a2eae93e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -37,6 +37,8 @@ public interface CalculatedFieldDao extends Dao { PageData findAll(PageLink pageLink); + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); boolean existsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index bed6f2d3a2..c0118d4f02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.cf; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -30,6 +32,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 677234dc20..fcbf4d4dd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -83,7 +83,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); calculatedField.setCreatedTime(createdTime); - calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); calculatedField.setType(type); calculatedField.setName(name); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index cdcffdd440..34bfa27b16 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -71,6 +71,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { From fd42c51df1699c371350c264f5e7116df0ebda94 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 11:25:22 +0200 Subject: [PATCH 101/281] Calculated field add/edit basic implementation --- .../core/http/calculated-fields.service.ts | 56 +---- .../calculated-fields-table-config.ts | 74 ++++-- .../calculated-fields-table.component.ts | 23 +- ...lated-field-arguments-table.component.html | 106 +++++++++ ...lated-field-arguments-table.component.scss | 32 +++ ...culated-field-arguments-table.component.ts | 213 ++++++++++++++++++ .../calculated-field-dialog.component.html | 169 ++++++++++++++ .../calculated-field-dialog.component.ts | 122 ++++++++++ ...ulated-field-argument-panel.component.html | 198 ++++++++++++++++ ...ulated-field-argument-panel.component.scss | 29 +++ ...lculated-field-argument-panel.component.ts | 201 +++++++++++++++++ .../components/public-api.ts | 18 ++ .../home/components/home-components.module.ts | 19 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity/entity-autocomplete.component.html | 13 +- .../entity/entity-autocomplete.component.ts | 7 + .../entity-key-autocomplete.component.html | 39 ++++ .../entity-key-autocomplete.component.ts | 134 +++++++++++ .../shared/components/js-func.component.ts | 46 ++-- .../shared/models/calculated-field.models.ts | 94 +++++++- ui-ngx/src/app/shared/models/public-api.ts | 1 + .../src/app/shared/models/regex.constants.ts | 17 ++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 67 +++++- 24 files changed, 1572 insertions(+), 117 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/regex.constants.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 5df4a84949..dca0c9a3c7 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable, of } from 'rxjs'; +import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; @@ -25,64 +25,26 @@ import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' }) -// [TODO]: [Calculated fields] - implement when BE ready export class CalculatedFieldsService { - fieldsMock = [ - { - name: 'Calculated Field 1', - type: 'Simple', - configuration: { - expression: '1 + 2', - type: 'SIMPLE', - }, - entityId: '1', - id: { - id: '1', - } - }, - { - name: 'Calculated Field 2', - type: 'Script', - entityId: '2', - configuration: { - expression: '${power}', - type: 'SIMPLE', - }, - id: { - id: '2', - } - } - ] as any[]; - constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + public getCalculatedFieldById(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { - return of(this.fieldsMock[1]); - // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); + public saveCalculatedField(calculatedField: CalculatedField, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(true); - // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, - config?: RequestConfig): Observable> { - return of({ - data: this.fieldsMock, - totalPages: 1, - totalElements: 2, - hasNext: false, - }); - // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, - // defaultHttpOptionsFromConfig(config)); + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 22f742b9d8..2500c92336 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -23,41 +23,33 @@ import { TimePageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; -import { DialogService } from '@core/services/dialog.service'; import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { getCurrentAuthState } from '@core/auth/auth.selectors'; -import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; -import { EntityService } from '@core/http/entity.service'; +import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { DestroyRef } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + readonly tenantId = getCurrentAuthUser(this.store).tenantId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, public entityId: EntityId = null, private store: Store, - private viewContainerRef: ViewContainerRef, - private overlay: Overlay, - private cd: ChangeDetectorRef, - private utilsService: UtilsService, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef, @@ -67,6 +59,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.addEntity = this.addCalculatedField.bind(this); + this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); + this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); + this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); + this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; @@ -97,8 +96,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - // // [TODO]: [Calculated fields] - implement edit - onAction: (_, entity) => {} + onAction: (_, entity) => this.editCalculatedField(entity) } ); } @@ -121,7 +119,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private editCalculatedField(calculatedField: CalculatedField): void { + this.getCalculatedFieldDialog(calculatedField, 'action.apply') + .afterClosed() + .pipe( + filter(Boolean), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private getCalculatedFieldDialog(value = {}, buttonTitle = 'action.add') { + return this.dialog.open(CalculatedFieldDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + value, + buttonTitle, + entityId: this.entityId, + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + tenantId: this.tenantId, + } + }) + } + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); @@ -149,7 +189,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), catchError(() => of(null)), takeUntilDestroyed(this.destroyRef), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index f98d24bd52..853870982a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,24 +16,18 @@ import { ChangeDetectionStrategy, - ChangeDetectorRef, Component, DestroyRef, Input, OnInit, ViewChild, - ViewContainerRef, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; -import { EntityService } from '@core/http/entity.service'; -import { DialogService } from '@core/services/dialog.service'; import { TranslateService } from '@ngx-translate/core'; import { MatDialog } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { TbPopoverService } from '@shared/components/popover.service'; @@ -51,7 +45,6 @@ export class CalculatedFieldsTableComponent implements OnInit { set entityId(entityId: EntityId) { if (this.entityIdValue !== entityId) { this.entityIdValue = entityId; - this.entitiesTable.resetSortAndFilter(this.activeValue); if (!this.activeValue) { this.hasInitialized = true; } @@ -78,18 +71,12 @@ export class CalculatedFieldsTableComponent implements OnInit { private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, private store: Store, - private overlay: Overlay, - private viewContainerRef: ViewContainerRef, - private cd: ChangeDetectorRef, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, - private destroyRef: DestroyRef, - private utilsService: UtilsService) { + private destroyRef: DestroyRef) { } ngOnInit() { @@ -97,19 +84,13 @@ export class CalculatedFieldsTableComponent implements OnInit { this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, - this.entityService, - this.dialogService, this.translate, this.dialog, this.entityIdValue, this.store, - this.viewContainerRef, - this.overlay, - this.cd, - this.utilsService, this.durationLeft, this.popoverService, - this.destroyRef + this.destroyRef, ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html new file mode 100644 index 0000000000..99e68cf059 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -0,0 +1,106 @@ + +

+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate }} + + + + } + + + + + {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + + + +
+ {{group.get('refEntityKey').get('key').value}} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
+ @if (errorText) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss new file mode 100644 index 0000000000..9507d9f012 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .inline-entity-autocomplete { + .mat-mdc-form-field-infix { + padding-top: 8px; + padding-bottom: 8px; + min-height: 40px; + width: auto; + .mdc-text-field__input, .mat-mdc-select { + font-weight: 400; + line-height: 20px; + } + } + a { + font-size: 14px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts new file mode 100644 index 0000000000..0ff6ecdf7f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -0,0 +1,213 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + DestroyRef, + effect, + forwardRef, + input, + Input, + Renderer2, + ViewContainerRef, +} from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { + ArgumentEntityType, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgument, + CalculatedFieldArgumentValue, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; + +@Component({ + selector: 'tb-calculated-field-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { + + @Input() entityId: EntityId; + @Input() tenantId: string; + + calculatedFieldType = input() + + errorText = ''; + argumentsFormArray = this.fb.array([]); + keysPopupClosed = true; + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly EntityType = EntityType; + readonly ArgumentEntityType = ArgumentEntityType; + + private onChange: (argumentsObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private destroyRef: DestroyRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2 + ) { + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { + this.onChange(this.getArgumentsObject()); + }); + effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + } + + registerOnChange(fn: (argumentsObj: Record) => void): void { + this.onChange = fn; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + return this.errorText ? { argumentsFormArray: false } : null; + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + + manageArgument($event: Event, matButton: MatButton, index?: number): void { + $event?.stopPropagation(); + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const ctx = { + index, + argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + entityId: this.entityId, + calculatedFieldType: this.calculatedFieldType(), + buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', + tenantId: this.tenantId, + }; + this.keysPopupClosed = false; + const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + ctx, + {}, + {}, {}, true); + argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; + argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + argumentsPanelPopover.hide(); + const formGroup = this.getArgumentFormGroup(value); + if (isDefinedAndNotNull(index)) { + this.argumentsFormArray.setControl(index, formGroup); + } else { + this.argumentsFormArray.push(formGroup); + } + this.argumentsFormArray.markAsDirty(); + this.cd.markForCheck(); + }); + argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.keysPopupClosed = true; + }); + } + } + + getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + return this.fb.group({ + ...value, + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...(value.refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: value.refEntityId.entityType, disabled: true }], + id: [{ value: value.refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: value.refEntityKey.type, disabled: true }], + key: [{ value: value.refEntityKey.key, disabled: true }], + }), + }) + } + + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html new file mode 100644 index 0000000000..7d1373bebb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -0,0 +1,169 @@ + +
+ +

{{ 'entity.type-calculated-field' | translate}}

+ +
+ + +
+
+
+
{{ 'common.general' | translate }}
+
+ + {{ 'entity-field.title' | translate }} + + @if (fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched) { + + @if (fieldFormGroup.get('name').hasError('required')) { + {{ 'common.hint.name-required' | translate }} + } @else if (fieldFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.name-pattern' | translate }} + } @else if (fieldFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.name-max-length' | translate }} + } + + } + + +
+ + {{ 'common.type' | translate }} + + @for (type of fieldTypes; track type) { + {{ CalculatedFieldTypeTranslations.get(type) | translate}} + } + + +
+ +
+
{{ 'calculated-fields.arguments' | translate }}
+ +
+
+
{{ 'calculated-fields.expression' | translate }}*
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { + + @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } + + } @else { + + } +
+
+
{{ 'calculated-fields.output' | translate }}
+
+ + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate}} + } + + + @if (outputFormGroup.get('type').value === OutputType.Attribute) { + + {{ 'calculated-fields.output-type' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if (data.entityId.entityType === EntityType.DEVICE) { + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + + } +
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + } +
+ +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts new file mode 100644 index 0000000000..99bf62ad46 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -0,0 +1,122 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { helpBaseUrl } from '@shared/models/constants'; +import { + CalculatedField, + CalculatedFieldConfiguration, + CalculatedFieldDialogData, + CalculatedFieldType, + CalculatedFieldTypeTranslations, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { map } from 'rxjs/operators'; +import { isObject } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; + +@Component({ + selector: 'tb-calculated-field-dialog', + templateUrl: './calculated-field-dialog.component.html', +}) +export class CalculatedFieldDialogComponent extends DialogComponent { + + fieldFormGroup = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + type: [CalculatedFieldType.SIMPLE, [Validators.required]], + debugSettings: [], + configuration: this.fb.group({ + arguments: [{}], + expressionSIMPLE: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + expressionSCRIPT: [], + output: this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + type: [OutputType.Timeseries] + }), + }), + }); + + functionArgs$ = this.fieldFormGroup.get('configuration').valueChanges + .pipe( + map(configuration => isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + ); + + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly EntityType = EntityType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly ScriptLanguage = ScriptLanguage; + readonly helpLink = `${helpBaseUrl}/[TODO: ADD VALID LINK!!!]`; + readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, + public dialogRef: MatDialogRef, + public fb: UntypedFormBuilder) { + super(store, router, dialogRef); + this.applyDialogData(); + this.outputFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleScopeByOutputType(type)); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + } + + get configFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration') as FormGroup; + } + + get outputFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration').get('output') as FormGroup; + } + + cancel(): void { + this.dialogRef.close(null); + } + + add(): void { + if (this.fieldFormGroup.valid) { + const { configuration, type, ...rest } = this.fieldFormGroup.value; + const { expressionSIMPLE, expressionSCRIPT, ...restConfig } = configuration; + this.dialogRef.close({ configuration: { ...restConfig, type, expression: configuration['expression'+type] }, ...rest, type }); + } + } + + private applyDialogData(): void { + const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value; + const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; + this.fieldFormGroup.patchValue({ configuration: { ...restConfig, ['expression'+type]: expression }, ...value }); + } + + private toggleScopeByOutputType(type: OutputType): void { + this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html new file mode 100644 index 0000000000..26a347aaa2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -0,0 +1,198 @@ + +
+
+
{{ 'calculated-fields.argument-settings' | translate }}
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
+ + + @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { + + warning + + } + +
+
+ +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + +
+ @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { +
+
{{ 'calculated-fields.device-name' | translate }}
+ +
+ } @else if (entityType === ArgumentEntityType.Customer) { +
+
{{ 'calculated-fields.customer-name' | translate }}
+ +
+ } + + +
+
{{ 'calculated-fields.argument-type' | translate }}
+ + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + + } + +
+ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } + + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { +
+
{{ 'calculated-fields.default-value' | translate }}
+
+ + + +
+
+ } @else { +
+
{{ 'calculated-fields.time-window' | translate }}
+
+ + + {{ 'common.suffix.ms' | translate }} + +
+
+
+
{{ 'calculated-fields.limit' | translate }}
+
+ + + +
+
+ } +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..a784909b92 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .mat-mdc-form-field { + width: 100%; + } +} + +:host ::ng-deep { + .entity-autocomplete { + .mat-mdc-form-field { + width: 100%; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts new file mode 100644 index 0000000000..2ba38beee5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -0,0 +1,201 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { + ArgumentEntityType, + ArgumentEntityTypeTranslations, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentValue, + CalculatedFieldType +} from '@shared/models/calculated-field.models'; +import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DatasourceType } from '@shared/models/widget.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { merge } from 'rxjs'; + +@Component({ + selector: 'tb-calculated-field-argument-panel', + templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] +}) +export class CalculatedFieldArgumentPanelComponent extends PageComponent implements OnInit { + + @Input() popover: TbPopoverComponent; + @Input() buttonTitle: string; + @Input() index: number; + @Input() argument: CalculatedFieldArgumentValue; + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() calculatedFieldType: CalculatedFieldType; + + @ViewChild('timeseriesInput') timeseriesInput: ElementRef; + + argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); + + argumentFormGroup = this.fb.group({ + argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + refEntityId: this.fb.group({ + entityType: [ArgumentEntityType.Current], + id: [''] + }), + refEntityKey: this.fb.group({ + type: [ArgumentType.LatestTelemetry, [Validators.required]], + key: [''], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + }), + defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], + limit: [null], + timeWindow: [null], + }); + + argumentTypes: ArgumentType[]; + entityFilter: EntityFilter; + keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges + .pipe( + startWith(this.refEntityIdFormGroup.get('entityType').value), + map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) + ); + + readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; + readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly DataKeyType = DataKeyType; + readonly EntityType = EntityType; + readonly datasourceType = DatasourceType; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly AttributeScope = AttributeScope; + + constructor( + private fb: FormBuilder, + ) { + super(); + + this.observeEntityFilterChanges(); + this.observeEntityTypeChanges() + this.observeEntityKeyChanges(); + } + + get entityType(): ArgumentEntityType { + return this.argumentFormGroup.get('refEntityId').get('entityType').value; + } + + get refEntityIdFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityId') as FormGroup; + } + + get refEntityKeyFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityKey') as FormGroup; + } + + ngOnInit(): void { + this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.toggleByEntityKeyType(this.argument.refEntityKey?.type); + this.setInitialEntityKeyType(); + + this.argumentTypes = Object.values(ArgumentType) + .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + } + + saveArgument(): void { + this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + } + + cancel(): void { + this.popover.hide(); + } + + private toggleByEntityKeyType(type: ArgumentType): void { + const isAttribute = type === ArgumentType.Attribute; + const isRolling = type === ArgumentType.Rolling; + this.argumentFormGroup.get('refEntityKey').get('scope')[isAttribute? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('limit')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('timeWindow')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); + } + + private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + let entityId: EntityId; + switch (entityType) { + case ArgumentEntityType.Current: + entityId = this.entityId + break; + case ArgumentEntityType.Tenant: + entityId = { + id: this.tenantId, + entityType: EntityType.TENANT + }; + break; + default: + entityId = this.argumentFormGroup.get('refEntityId').value as any; + } + if (onInit) { + this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); + } + this.entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } + + private observeEntityFilterChanges(): void { + merge( + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges, + this.argumentFormGroup.get('refEntityId').get('id').valueChanges.pipe(filter(Boolean)), + this.argumentFormGroup.get('refEntityKey').get('scope').valueChanges, + ) + .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .subscribe(() => this.updateEntityFilter(this.entityType)); + } + + private observeEntityTypeChanges(): void { + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges + .pipe(distinctUntilChanged(), takeUntilDestroyed()) + .subscribe(type => { + this.argumentFormGroup.get('refEntityId').get('id').setValue(''); + this.argumentFormGroup.get('refEntityId') + .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + }); + } + + private observeEntityKeyChanges(): void { + this.argumentFormGroup.get('refEntityKey').get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleByEntityKeyType(type)); + } + + private setInitialEntityKeyType(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); + typeControl.setValue(null); + typeControl.markAsTouched(); + typeControl.updateValueAndValidity(); + } + } + + protected readonly ArgumentEntityType = ArgumentEntityType; +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts new file mode 100644 index 0000000000..c3d1ede02e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -0,0 +1,18 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export * from './dialog/calculated-field-dialog.component'; +export * from './arguments-table/calculated-field-arguments-table.component'; diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 1a6b9c08e0..f60ea15407 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -185,6 +185,16 @@ import { EntityChipsComponent } from '@home/components/entity/entity-chips.compo import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; @NgModule({ declarations: @@ -330,6 +340,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], imports: [ CommonModule, @@ -341,7 +354,8 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; SnmpDeviceProfileTransportModule, StatesControllerModule, DeviceCredentialsModule, - DeviceProfileCommonModule + DeviceProfileCommonModule, + EntityDebugSettingsButtonComponent ], exports: [ RouterTabsComponent, @@ -468,6 +482,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts index 618650ac32..fbbf124629 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts @@ -19,6 +19,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DeviceInfo } from '@shared/models/device.models'; import { EntityTabsComponent } from '../../components/entity/entity-tabs.component'; +import { EntityType } from '@shared/models/entity-type.models'; @Component({ selector: 'tb-device-tabs', @@ -27,6 +28,8 @@ import { EntityTabsComponent } from '../../components/entity/entity-tabs.compone }) export class DeviceTabsComponent extends EntityTabsComponent { + readonly EntityType = EntityType; + constructor(protected store: Store) { super(store); } @@ -34,5 +37,4 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } - } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index c194e04290..44a218b455 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -17,10 +17,11 @@ --> - {{ label | translate }} + {{ label | translate }} {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} + + warning + + } + + @for (key of filteredKeys$ | async; track key) { + + } + + diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts new file mode 100644 index 0000000000..2a33a74786 --- /dev/null +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -0,0 +1,134 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, effect, ElementRef, forwardRef, input, ViewChild, } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { combineLatest, of, Subject } from 'rxjs'; +import { EntityService } from '@core/http/entity.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntitiesKeysByQuery } from '@shared/models/entity.models'; +import { EntityFilter } from '@shared/models/query/query.models'; + +@Component({ + selector: 'tb-entity-key-autocomplete', + templateUrl: './entity-key-autocomplete.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + } + ], + host: { + class: 'w-full' + } +}) +export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { + + @ViewChild('keyInput') keyInput: ElementRef; + + entityFilter = input.required(); + dataKeyType = input.required(); + keyScopeType = input(); + + keyControl = this.fb.control('', [Validators.required]); + searchText = ''; + keyInputSubject = new Subject(); + + private onChange: (value: string) => void; + private cachedResult: EntitiesKeysByQuery; + + keys$ = this.keyInputSubject.asObservable() + .pipe( + switchMap(() => { + return this.cachedResult ? of(this.cachedResult) : this.entityService.findEntityKeysByQuery({ + pageLink: { page: 0, pageSize: 100 }, + entityFilter: this.entityFilter(), + }, this.dataKeyType() === DataKeyType.attribute, this.dataKeyType() === DataKeyType.timeseries, this.keyScopeType()); + }), + map(result => { + this.cachedResult = result; + switch (this.dataKeyType()) { + case DataKeyType.attribute: + return result.attribute; + case DataKeyType.timeseries: + return result.timeseries; + default: + return []; + } + }), + ); + + filteredKeys$ = combineLatest([this.keys$, this.keyControl.valueChanges.pipe(startWith(''))]) + .pipe( + map(([keys, searchText = '']) => { + this.searchText = searchText; + return searchText ? keys.filter(item => item.toLowerCase().includes(searchText.toLowerCase())) : keys; + }) + ); + + constructor( + private fb: FormBuilder, + private entityService: EntityService, + ) { + this.keyControl.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.onChange(value)); + effect(() => { + if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { + this.cachedResult = null; + this.searchText = ''; + } + }); + } + + clear(): void { + this.keyControl.patchValue('', {emitEvent: true}); + setTimeout(() => { + this.keyInput.nativeElement.blur(); + this.keyInput.nativeElement.focus(); + }, 0); + } + + registerOnChange(onChange: (value: string) => void): void { + this.onChange = onChange; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + return this.keyControl.valid ? null : { keyControl: false }; + } + + writeValue(value: string): void { + this.keyControl.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 8e500bdbe9..57c15a5484 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -20,9 +20,11 @@ import { ElementRef, forwardRef, Input, + OnChanges, OnDestroy, OnInit, Renderer2, + SimpleChanges, ViewChild, ViewContainerRef, ViewEncapsulation @@ -67,7 +69,7 @@ import { catchError } from 'rxjs/operators'; ], encapsulation: ViewEncapsulation.None }) -export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { +export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlValueAccessor, Validator { @ViewChild('javascriptEditor', {static: true}) javascriptEditorElmRef: ElementRef; @@ -177,6 +179,13 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, private http: HttpClient) { } + ngOnChanges(changes: SimpleChanges): void { + if (changes.functionArgs) { + this.updateFunctionArgsString(); + this.updateFunctionLabel(); + } + } + ngOnInit(): void { if (this.functionTitle || this.label) { this.hideBrackets = true; @@ -184,22 +193,6 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } - if (this.functionArgs) { - this.functionArgs.forEach((functionArg) => { - if (this.functionArgsString.length > 0) { - this.functionArgsString += ', '; - } - this.functionArgsString += functionArg; - }); - } - if (this.functionTitle) { - this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; - } else if (this.label) { - this.functionLabel = this.label; - } else { - this.functionLabel = - `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; - } const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -329,6 +322,25 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + private updateFunctionArgsString(): void { + this.functionArgsString = ''; + if (this.functionArgs) { + this.functionArgsString = this.functionArgs.join(', '); + } + } + + private updateFunctionLabel(): void { + if (this.functionTitle) { + this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; + } else if (this.label) { + this.functionLabel = this.label; + } else { + this.functionLabel = + `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; + } + this.cd.markForCheck(); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 253bc58f39..73b3ecd861 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -13,29 +13,109 @@ /// See the License for the specific language governing permissions and /// limitations under the License. /// + import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; +import { EntityId } from '@shared/models/id/entity-id'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { - type: CalculatedFieldType; debugSettings?: EntityDebugSettings; externalId?: string; configuration: CalculatedFieldConfiguration; + type: CalculatedFieldType; } export enum CalculatedFieldType { SIMPLE = 'SIMPLE', - COMPLEX = 'COMPLEX', + SCRIPT = 'SCRIPT', } +export const CalculatedFieldTypeTranslations = new Map( + [ + [CalculatedFieldType.SIMPLE, 'calculated-fields.type.simple'], + [CalculatedFieldType.SCRIPT, 'calculated-fields.type.script'], + ] +) + export interface CalculatedFieldConfiguration { - type: CalculatedFieldConfigType; + type: CalculatedFieldType; expression: string; - arguments: Record; + arguments: Record; } -export enum CalculatedFieldConfigType { - SIMPLE = 'SIMPLE', - SCRIPT = 'SCRIPT', +export enum ArgumentEntityType { + Current = 'CURRENT', + Device = 'DEVICE', + Asset = 'ASSET', + Customer = 'CUSTOMER', + Tenant = 'TENANT', +} + +export const ArgumentEntityTypeTranslations = new Map( + [ + [ArgumentEntityType.Current, 'calculated-fields.argument-current'], + [ArgumentEntityType.Device, 'calculated-fields.argument-device'], + [ArgumentEntityType.Asset, 'calculated-fields.argument-asset'], + [ArgumentEntityType.Customer, 'calculated-fields.argument-customer'], + [ArgumentEntityType.Tenant, 'calculated-fields.argument-tenant'], + ] +) + +export enum ArgumentType { + Attribute = 'ATTRIBUTE', + LatestTelemetry = 'TS_LATEST', + Rolling = 'TS_ROLLING', +} + +export enum OutputType { + Attribute = 'ATTRIBUTES', + Timeseries = 'TIME_SERIES', +} + +export const OutputTypeTranslations = new Map( + [ + [OutputType.Attribute, 'calculated-fields.attribute'], + [OutputType.Timeseries, 'calculated-fields.timeseries'], + ] +) + +export const ArgumentTypeTranslations = new Map( + [ + [ArgumentType.Attribute, 'calculated-fields.attribute'], + [ArgumentType.LatestTelemetry, 'calculated-fields.latest-telemetry'], + [ArgumentType.Rolling, 'calculated-fields.rolling'], + ] +) + +export interface CalculatedFieldArgument { + refEntityKey: RefEntityKey; + defaultValue?: string; + refEntityId?: RefEntityKey; + limit?: number; + timeWindow?: number; +} + +export interface RefEntityKey { + key: string; + type: ArgumentType; + scope?: AttributeScope; +} + +export interface RefEntityKey { + entityType: ArgumentEntityType; + id: string; +} + +export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { + argumentName: string; +} + +export interface CalculatedFieldDialogData { + value: CalculatedField; + buttonTitle: string; + entityId: EntityId; + debugLimitsConfiguration: string; + tenantId: string; } diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 736d885810..c48a3286cd 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -61,3 +61,4 @@ export * from './widgets-bundle.model'; export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; +export * from './regex.constants'; diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts new file mode 100644 index 0000000000..55742cefd4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -0,0 +1,17 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 5b40dadf5f..5825998434 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -224,6 +224,7 @@ import { IntervalOptionsConfigPanelComponent } from '@shared/components/time/int import { GroupingIntervalOptionsComponent } from '@shared/components/time/aggregation/grouping-interval-options.component'; import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component'; import { JsFuncModuleRowComponent } from '@shared/components/js-func-module-row.component'; +import { EntityKeyAutocompleteComponent } from '@shared/components/entity/entity-key-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -432,7 +433,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) ImageGalleryDialogComponent, WidgetButtonComponent, HexInputComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ], imports: [ CommonModule, @@ -694,7 +696,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index beb6734451..be31b4f70b 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -996,6 +996,7 @@ "failures": "Failures", "entity": "entity", "rule-node": "rule node", + "calculated-field": "calculated field", "hint": { "main": "All node debug messages rate limited with:", "main-limited": "All {{entity}} debug messages will be rate-limited, with a maximum of {{msg}} messages allowed per {{time}}.", @@ -1007,7 +1008,56 @@ "expression": "Expression", "no-found": "No calculated fields found", "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", - "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected" + "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", + "type": { + "simple": "Simple", + "script": "Script" + }, + "arguments": "Arguments", + "argument-name": "Argument name", + "datasource": "Datasource", + "add-argument": "Add argument", + "no-arguments": "No arguments configured", + "argument-settings": "Argument settings", + "argument-current": "Current entity", + "argument-current-tenant": "Current tenant", + "argument-device": "Device", + "argument-asset": "Asset", + "argument-customer": "Customer", + "argument-tenant": "Current tenant", + "argument-type": "Argument type", + "attribute": "Attribute", + "timeseries-key": "Time series key", + "device-name": "Device name", + "latest-telemetry": "Latest telemetry", + "rolling": "Rolling", + "attribute-scope": "Attribute scope", + "server-attributes": "Server attributes", + "client-attributes": "Client attributes", + "shared-attributes": "Shared attributes", + "attribute-key": "Attribute key", + "default-value": "Default value", + "limit": "Limit", + "time-window": "Time window", + "customer-name": "Customer name", + "timeseries": "Time series", + "output": "Output", + "output-type": "Output type", + "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", + "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", + "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", + "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", + "hint": { + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-empty": "Arguments should not be empty.", + "expression-required": "Expression is required.", + "expression-invalid": "Expression is invalid", + "expression-max-length": "Expression length should be less than 255 characters.", + "argument-name-required": "Argument name is required.", + "argument-name-pattern": "Argument name is invalid.", + "argument-name-max-length": "Argument name should be less than 256 characters.", + "argument-type-required": "Argument type is required." + } }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", @@ -1035,6 +1085,7 @@ "common": { "name": "Name", "type": "Type", + "general": "General", "username": "Username", "password": "Password", "enter-username": "Enter username", @@ -1047,7 +1098,19 @@ "open-details-page": "Open details page", "not-found": "Not found", "documentation": "Documentation", - "time-left": "{{time}} left" + "time-left": "{{time}} left", + "suffix": { + "s": "s", + "ms": "ms" + }, + "hint": { + "name-required": "Name is required.", + "name-pattern": "Name is invalid.", + "name-max-length": "Name should be less than 256 characters.", + "key-required": "Key is required.", + "key-pattern": "Key is invalid.", + "key-max-length": "Key should be less than 256 characters." + } }, "content-type": { "json": "Json", From c1c9fb4f5aa9ee4fa5c369ba51436b194d908b1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 11:29:38 +0200 Subject: [PATCH 102/281] implemented calculated field debug events persistence --- .../server/actors/ActorSystemContext.java | 52 +++++++++++-------- ...CalculatedFieldEntityMessageProcessor.java | 42 ++++++++++----- ...alculatedFieldManagerMessageProcessor.java | 20 +++---- ...efaultCalculatedFieldExecutionService.java | 15 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 19 ++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 42 +++++++++++++-- .../cf/ctx/state/CalculatedFieldState.java | 4 ++ .../ctx/state/ScriptCalculatedFieldState.java | 6 +++ .../ctx/state/SimpleCalculatedFieldState.java | 15 ++---- .../cf/DefaultTbCalculatedFieldService.java | 10 ++++ common/proto/src/main/proto/queue.proto | 3 ++ ...efaultNativeCalculatedFieldRepository.java | 3 ++ .../engine/api/AttributesSaveRequest.java | 18 ++++++- .../engine/api/TimeseriesSaveRequest.java | 18 ++++++- .../engine/telemetry/TbMsgAttributesNode.java | 2 + .../engine/telemetry/TbMsgTimeseriesNode.java | 2 + 16 files changed, 194 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 09488bfe4e..daa6744fd9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; @@ -105,6 +106,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -129,6 +131,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -744,29 +747,34 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { - if (checkLimits(tenantId, tbMsg, error)) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(tbMsg.getOriginator()) - .msgId(tbMsg.getId()) - .msgType(tbMsg.getType()) - .arguments(JacksonUtil.toString(arguments)) - .result(tbMsg.getData()); - - if (error != null) { - event.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(event.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString(arguments)); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..6426be8a3c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,19 +18,18 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -53,9 +52,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** @@ -111,9 +108,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(CALLBACKS_PER_CF); } else { if (proto.getTsDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getAttrDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -136,27 +133,30 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) { + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + } } else { callback.onSuccess(); // State was updated but no calculation performed; } @@ -183,13 +183,27 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +235,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..603ab96a8e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,14 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -38,16 +36,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -101,6 +90,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9c8aafc47d..8f948aa265 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -833,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); List entries = request.getEntries(); List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { @@ -849,7 +849,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); List entries = request.getEntries(); for (int i = 0; i < entries.size(); i++) { @@ -862,7 +862,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); @@ -878,6 +878,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + return telemetryMsg; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d623043cee..f5f681b505 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,19 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.NoArgsConstructor; + import java.util.HashMap; import java.util.List; import java.util.Map; +@NoArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; protected Map arguments; - public BaseCalculatedFieldState() { - this.arguments = new HashMap<>(); - } - public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; this.arguments = new HashMap<>(); @@ -35,7 +34,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public Map getArguments() { - return this.arguments; + return arguments; + } + + @Override + public List getRequiredArguments() { + return requiredArguments; } @Override @@ -53,7 +57,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingEntry == null) { validateNewEntry(newEntry); - arguments.put(key, newEntry.copy()); + arguments.put(key, newEntry); stateUpdated = true; } else { stateUpdated = existingEntry.updateEntry(newEntry); @@ -70,7 +74,6 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } - protected void validateNewEntry(ArgumentEntry newEntry) { - } + protected abstract void validateNewEntry(ArgumentEntry newEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 42536188ce..33421a7d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; +import org.mvel2.MVEL; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -33,7 +35,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -45,6 +46,8 @@ import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { + private CalculatedField calculatedField; + private CalculatedFieldId cfId; private TenantId tenantId; private EntityId entityId; @@ -61,7 +64,11 @@ public class CalculatedFieldCtx { private CalculatedFieldScriptEngine calculatedFieldScriptEngine; private ThreadLocal customExpression; + private boolean initialized; + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.calculatedField = calculatedField; + this.cfId = calculatedField.getId(); this.tenantId = calculatedField.getTenantId(); this.entityId = calculatedField.getEntityId(); @@ -88,10 +95,28 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { - this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + + public void init() { + if (CalculatedFieldType.SCRIPT.equals(cfType)) { + try { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } } else { - this.customExpression = new ThreadLocal<>(); + if (isValidExpression(expression)) { + this.customExpression = ThreadLocal.withInitial(() -> + new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build() + ); + initialized = true; + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + } } } @@ -108,6 +133,15 @@ public class CalculatedFieldCtx { ); } + private boolean isValidExpression(String expression) { + try { + MVEL.compileExpression(expression); + return true; + } catch (Exception e) { + return false; + } + } + public boolean matches(List values, AttributeScope scope) { return matchesAttributes(mainEntityArguments, values, scope); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..173d299da6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; @JsonTypeInfo( @@ -40,9 +41,12 @@ public interface CalculatedFieldState { Map getArguments(); + List getRequiredArguments(); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 290fd95370..a3c7efb388 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -31,6 +32,7 @@ import java.util.TreeMap; @Data @Slf4j +@NoArgsConstructor public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ScriptCalculatedFieldState(List requiredArguments) { @@ -42,6 +44,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 59b3009bb4..8b8fe6e8c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -18,8 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import net.objecthunter.exp4j.Expression; -import net.objecthunter.exp4j.ExpressionBuilder; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -28,6 +27,7 @@ import java.util.List; import java.util.Map; @Data +@NoArgsConstructor public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { public SimpleCalculatedFieldState(List requiredArguments) { @@ -48,16 +48,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = ctx.getCustomExpression(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } + var expr = ctx.getCustomExpression().get(); for (Map.Entry entry : this.arguments.entrySet()) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index c8c589691b..184092720c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -58,6 +58,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + if (ActionType.UPDATED.equals(actionType)) { + CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); + checkForEntityChange(existingCf, calculatedField); + } checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); checkArgumentSize(calculatedField.getConfiguration()); @@ -98,6 +102,12 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { + throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); + } + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..53045fc147 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -799,6 +799,9 @@ message CalculatedFieldTelemetryMsgProto { repeated TsKvProto tsData = 9; AttributeScopeProto scope = 10; repeated AttributeValueProto attrData = 11; + int64 tbMsgIdMSB = 12; + int64 tbMsgIdLSB = 13; + string tbMsgType = 14; } message CalculatedFieldLinkedTelemetryMsgProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fcbf4d4dd0..daca6d5056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -78,6 +79,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; + String debugSettings = (String) row.get("debug_settings"); Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); @@ -90,6 +92,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); return calculatedField; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 406b2d0d5c..74f3468f49 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -28,8 +28,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @@ -42,6 +44,8 @@ public class AttributesSaveRequest { private final List entries; private final boolean notifyDevice; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +60,8 @@ public class AttributesSaveRequest { private List entries; private boolean notifyDevice = true; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -108,6 +114,16 @@ public class AttributesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -128,7 +144,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 95eb788e5f..957b0cf108 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -27,8 +27,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) @@ -42,6 +44,8 @@ public class TimeseriesSaveRequest { private final boolean saveLatest; private final boolean onlyLatest; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -59,6 +63,8 @@ public class TimeseriesSaveRequest { private boolean saveLatest = true; private boolean onlyLatest; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; Builder() {} @@ -111,6 +117,16 @@ public class TimeseriesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -131,7 +147,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index ae2fce6575..925d70daa1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -126,6 +126,8 @@ public class TbMsgAttributesNode implements TbNode { .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 89f7844313..3287cc825f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -113,6 +113,8 @@ public class TbMsgTimeseriesNode implements TbNode { .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From c01024f6da8b66b2a78adcde6884d40bfd15cabc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 12:11:09 +0200 Subject: [PATCH 103/281] changed endpoint --- .../server/controller/CalculatedFieldController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 124dd8d710..96a97aeeff 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -94,11 +94,11 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + @ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody public PageData getCalculatedFieldsByEntityId( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, From a82d3690f3e282bb3ef18aec90bda15f69a387ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 12:30:36 +0200 Subject: [PATCH 104/281] Changed getAll endpoint and refactoring --- .../core/http/calculated-fields.service.ts | 5 +- .../calculated-fields-table-config.ts | 16 +- ...lated-field-arguments-table.component.html | 138 +++++++++--------- ...lated-field-arguments-table.component.scss | 1 + ...culated-field-arguments-table.component.ts | 101 +++++++------ .../calculated-field-dialog.component.html | 4 +- .../calculated-field-dialog.component.ts | 4 +- ...ulated-field-argument-panel.component.html | 53 +++---- ...lculated-field-argument-panel.component.ts | 9 +- .../components/public-api.ts | 1 + .../pages/device/device-tabs.component.html | 3 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 9 +- .../shared/models/calculated-field.models.ts | 1 + 14 files changed, 179 insertions(+), 170 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index dca0c9a3c7..10ad366e5b 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -21,6 +21,7 @@ import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; +import { EntityId } from '@shared/models/id/entity-id'; @Injectable({ providedIn: 'root' @@ -43,8 +44,8 @@ export class CalculatedFieldsService { return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { - return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + public getCalculatedFields({ entityType, id }: EntityId, pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 2500c92336..ae025a13f6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -40,6 +40,7 @@ import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { + // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; @@ -66,9 +67,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: TimePageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); - this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); @@ -102,7 +103,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { - return this.calculatedFieldsService.getCalculatedFields(pageLink); + return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { @@ -134,10 +135,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) .subscribe((res) => { if (res) { @@ -148,10 +148,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), ) .subscribe((res) => { if (res) { @@ -160,7 +159,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -172,6 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig{{ 'common.type' | translate }}
{{ 'entity.key' | translate }}
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id').value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate }} - - - - } - +
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + - - - {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - -
- {{group.get('refEntityKey').get('key').value}} -
- - + -
- - -
+ } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } + + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ +
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
@if (errorText) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 9507d9f012..8695ee4068 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -25,6 +25,7 @@ line-height: 20px; } } + a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 0ff6ecdf7f..1dedc5e80e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -43,9 +43,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType, } from '@shared/models/calculated-field.models'; -import { - CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -87,7 +85,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; - private onChange: (argumentsObj: Record) => void = () => {}; + private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, @@ -98,59 +96,27 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private renderer: Renderer2 ) { this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { - this.onChange(this.getArgumentsObject()); + this.propagateChange(this.getArgumentsObject()); }); effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); } registerOnChange(fn: (argumentsObj: Record) => void): void { - this.onChange = fn; + this.propagateChange = fn; } registerOnTouched(_): void {} validate(): ValidationErrors | null { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE - && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { - this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; - } else if (!this.argumentsFormArray.controls.length) { - this.errorText = 'calculated-fields.hint.arguments-empty'; - } else { - this.errorText = ''; - } + this.updateErrorText(); return this.errorText ? { argumentsFormArray: false } : null; } - private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); - const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; - acc[argumentName] = argument; - return acc; - }, {} as Record); - } - - writeValue(argumentsObj: Record): void { - this.argumentsFormArray.clear(); - Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], - ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); - }); + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); } - manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); const trigger = matButton._elementRef.nativeElement; @@ -189,7 +155,51 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } - getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private updateErrorText(): void { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + this.populateArgumentsFormArray(argumentsObj) + } + + private populateArgumentsFormArray(argumentsObj: Record): void { + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], @@ -205,9 +215,4 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), }) } - - onDelete(index: number): void { - this.argumentsFormArray.removeAt(index); - this.argumentsFormArray.markAsDirty(); - } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 7d1373bebb..e6adc1b4d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -68,7 +68,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" - [calculatedFieldType]="fieldFormGroup.get('type').valueChanges | async" + [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -96,7 +96,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: ADD VALID LINK HERE!!!" + helpId="[TODO]: [Calculated Fields] add valid link" /> }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 99bf62ad46..7f9af5cead 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -47,7 +47,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent - @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { - - warning - + @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } @@ -118,7 +120,7 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
@@ -143,6 +145,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
this.updateEntityFilter(this.entityType)); @@ -196,6 +197,4 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme typeControl.updateValueAndValidity(); } } - - protected readonly ArgumentEntityType = ArgumentEntityType; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index c3d1ede02e..bc89e4dc6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -16,3 +16,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; +export * from './panel/calculated-field-argument-panel.component'; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index ab59f412cd..6a4f87ca6b 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,8 +32,7 @@ [entityName]="entity.name"> - + { - readonly EntityType = EntityType; - constructor(protected store: Store) { super(store); } @@ -37,4 +34,5 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } + } diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts index 2a33a74786..fc13238db0 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -47,9 +47,6 @@ import { EntityFilter } from '@shared/models/query/query.models'; multi: true } ], - host: { - class: 'w-full' - } }) export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { @@ -63,7 +60,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val searchText = ''; keyInputSubject = new Subject(); - private onChange: (value: string) => void; + private propagateChange: (value: string) => void; private cachedResult: EntitiesKeysByQuery; keys$ = this.keyInputSubject.asObservable() @@ -101,7 +98,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val ) { this.keyControl.valueChanges .pipe(takeUntilDestroyed()) - .subscribe(value => this.onChange(value)); + .subscribe(value => this.propagateChange(value)); effect(() => { if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { this.cachedResult = null; @@ -119,7 +116,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val } registerOnChange(onChange: (value: string) => void): void { - this.onChange = onChange; + this.propagateChange = onChange; } registerOnTouched(_): void {} diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 73b3ecd861..54798a23e7 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -25,6 +25,7 @@ export interface CalculatedField extends Omit, 'labe externalId?: string; configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; + entityId: EntityId; } export enum CalculatedFieldType { From d3216f3ee78ce154ea3a4c8a224003728497c84f Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 13:11:12 +0200 Subject: [PATCH 105/281] Changed table load --- .../calculated-fields-table.component.ts | 62 ++++++------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 853870982a..4c22ad2896 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -18,8 +18,8 @@ import { ChangeDetectionStrategy, Component, DestroyRef, - Input, - OnInit, + effect, + input, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -39,36 +39,14 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class CalculatedFieldsTableComponent implements OnInit { - - @Input() - set entityId(entityId: EntityId) { - if (this.entityIdValue !== entityId) { - this.entityIdValue = entityId; - if (!this.activeValue) { - this.hasInitialized = true; - } - } - } - - @Input() - set active(active: boolean) { - if (this.activeValue !== active) { - this.activeValue = active; - if (this.activeValue && this.hasInitialized) { - this.hasInitialized = false; - this.entitiesTable.updateData(); - } - } - } +export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + active = input(); + entityId = input(); - private activeValue = false; - private hasInitialized = false; - private entityIdValue: EntityId; + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; constructor(private calculatedFieldsService: CalculatedFieldsService, private translate: TranslateService, @@ -77,20 +55,20 @@ export class CalculatedFieldsTableComponent implements OnInit { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef) { - } - ngOnInit() { - this.hasInitialized = !this.activeValue; - - this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( - this.calculatedFieldsService, - this.translate, - this.dialog, - this.entityIdValue, - this.store, - this.durationLeft, - this.popoverService, - this.destroyRef, - ); + effect(() => { + if (this.active()) { + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.translate, + this.dialog, + this.entityId(), + this.store, + this.durationLeft, + this.popoverService, + this.destroyRef, + ); + } + }); } } From 83b338c697ff6ab7188886e273df4736dd1c3d80 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 31 Jan 2025 13:54:53 +0200 Subject: [PATCH 106/281] Entity lifecycle implementation --- .../server/actors/app/AppActor.java | 5 +- .../CalculatedFieldEntityActor.java | 6 + .../CalculatedFieldEntityDeleteMsg.java | 44 +++ ...CalculatedFieldEntityMessageProcessor.java | 51 ++- .../CalculatedFieldManagerActor.java | 8 +- ...alculatedFieldManagerMessageProcessor.java | 215 +++++++++++- .../EntityInitCalculatedFieldMsg.java | 41 +++ .../server/actors/tenant/TenantActor.java | 3 +- .../cf/CalculatedFieldExecutionService.java | 21 +- ...efaultCalculatedFieldExecutionService.java | 314 +++--------------- .../cf/ctx/CalculatedFieldStateService.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 9 +- .../cf/ctx/state/RocksDBStateService.java | 4 +- .../entitiy/EntityStateSourcingListener.java | 10 +- ...faultTbCalculatedFieldConsumerService.java | 100 +----- .../queue/DefaultTbClusterService.java | 150 ++++----- .../server/cluster/TbClusterService.java | 4 +- .../server/queue/TbQueueCallback.java | 14 + .../server/common/msg/MsgType.java | 14 +- .../cf/CalculatedFieldEntityLifecycleMsg.java | 37 +++ .../msg/plugin/ComponentLifecycleMsg.java | 20 ++ .../server/common/util/ProtoUtils.java | 41 ++- common/proto/src/main/proto/queue.proto | 35 +- 23 files changed, 595 insertions(+), 553 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 9124dc0a9a..8a6907c107 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -115,12 +115,13 @@ public class AppActor extends ContextAwareActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. + // same for the Linked telemetry. onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index a5461c6152..cebe6b6a60 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -53,6 +53,12 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; + case CF_ENTITY_INIT_CF_MSG: + processor.process((EntityInitCalculatedFieldMsg) msg); + break; + case CF_ENTITY_DELETE_MSG: + processor.process((CalculatedFieldEntityDeleteMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java new file mode 100644 index 0000000000..8df6a63a82 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final TbCallback callback; + + public CalculatedFieldEntityDeleteMsg(TenantId tenantId, + EntityId entityId, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_DELETE_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..b8f357fd87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -88,6 +89,29 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM states.put(msg.getId().cfId(), msg.getState()); } + public void process(EntityInitCalculatedFieldMsg msg) { + var cfCtx = msg.getCtx(); + if (msg.isForceReinit()) { + states.remove(cfCtx.getCfId()); + } + var cfState = getOrInitState(cfCtx); + processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, msg.getCallback()); + } + + public void process(CalculatedFieldEntityDeleteMsg msg) { + if (this.entityId.equals(msg.getEntityId())) { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } else { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = states.remove(cfId); + if (state != null) { + cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } + } + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -152,15 +176,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -183,13 +201,24 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + @SneakyThrows + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, TbCallback callback) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +250,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 1c198c660d..22909dd5af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; @@ -66,8 +67,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); break; - case CF_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + case CF_ENTITY_LIFECYCLE_MSG: + processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); break; case CF_TELEMETRY_MSG: processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); @@ -75,9 +76,6 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_LINKED_TELEMETRY_MSG: processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); break; - case CF_ENTITY_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); - break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..e82f407e3a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,13 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -36,18 +33,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -78,7 +70,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfService; + private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -88,7 +81,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -104,7 +98,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -112,7 +106,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link); msg.getCallback().onSuccess(); } @@ -124,6 +118,168 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + var entityType = msg.getData().getEntityId().getEntityType(); + var event = msg.getData().getEvent(); + switch (entityType) { + case CALCULATED_FIELD: { + switch (event) { + case CREATED: + onCfCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onCfUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onCfDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + case DEVICE: + case ASSET: { + switch (event) { + case CREATED: + onEntityCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onEntityUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onEntityDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + default: { + msg.getCallback().onSuccess(); + } + } + } + + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { + EntityId entityId = msg.getEntityId(); + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + var fieldsCount = entityIdFields.size() + profileIdFields.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + + private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); + var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); + var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + var entityId = msg.getEntityId(); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + } + + private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + if (calculatedFields.containsKey(cfId)) { + log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cf = cfDaoService.findById(msg.getTenantId(), cfId); + if (cf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + initCf(cfCtx, callback, false); + } + } + } + + private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var oldCfCtx = calculatedFields.get(cfId); + if (oldCfCtx == null) { + onCfCreated(msg, callback); + } else { + var newCf = cfDaoService.findById(msg.getTenantId(), cfId); + if (newCf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + calculatedFields.put(newCf.getId(), newCfCtx); + List oldCfList = entityIdCalculatedFields.get(newCf.getId()); + List newCfList = new ArrayList<>(oldCfList.size()); + boolean found = false; + for (CalculatedFieldCtx oldCtx : oldCfList) { + if (oldCtx.getCfId().equals(newCf.getId())) { + newCfList.add(newCfCtx); + found = true; + } else { + newCfList.add(oldCtx); + } + } + if (!found) { + newCfList.add(newCfCtx); + } + entityIdCalculatedFields.put(newCf.getId(), newCfList); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + initCf(newCfCtx, callback, true); + } + } + } + + } + + private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var cfCtx = calculatedFields.remove(cfId); + if (cfCtx == null) { + log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + //TODO: no need to do this if we cache all created actors and know which one belong to us; + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + deleteCfForEntity(entityId, cfId, callback); + } + } + } + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); // 2 = 1 for CF processing + 1 for links processing @@ -140,7 +296,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List linkedCalculatedFields = filterCalculatedFieldLinks(msg); var linksSize = linkedCalculatedFields.size(); if (linksSize > 0) { - cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); } else { callback.onSuccess(); } @@ -237,6 +393,33 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return entities; } + private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + initCfForEntity(entityId, cfCtx, forceStateReinit, callback); + } + } + + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); + } + + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + } + + private static boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java new file mode 100644 index 0000000000..00a378c90a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + private final boolean forceReinit; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_INIT_CF_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 6ae10aaece..1b008f462a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,12 +170,11 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d8daae2e64..19f60165cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,16 +20,11 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -50,25 +45,11 @@ public interface CalculatedFieldExecutionService { void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); -// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - /* ===================================================== */ - - void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - + void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 19561b73b9..9aca0d5b3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -65,7 +64,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -75,11 +73,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -122,7 +117,6 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -273,6 +267,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas stateService.persistState(stateId, state, callback); } + @Override + public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { + stateService.removeState(calculatedFieldEntityCtxId, callback); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -330,271 +329,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } - @Override - public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { - EntityId entityId = calculatedField.getEntityId(); - ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() - .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) - .collect(Collectors.toSet()); - tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); - } - default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() - + "' does not support calculated fields."); - } - } - - @Override - public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - ComponentLifecycleEvent event = proto.getEvent(); - if (ComponentLifecycleEvent.DELETED.equals(event)) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.evict(calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (ComponentLifecycleEvent.UPDATED.equals(event)) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - if (cf != null) { - calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); - EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(calculatedFieldCtx, entityId, callback); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); - }); - }); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); - } else { - calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> { - if (ctxId.cfId().equals(calculatedFieldId)) { - stateService.removeState(ctxId); - return true; - } - return false; - }); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - - return entityIdChanged || typeChanged || argumentsChanged; - } - - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); - if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); - } - } - - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto.getMsg(), callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - -// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - - @Override - public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - } - if (proto.getAdded()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - if (proto.getUpdated()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - } else { - clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity update msg: [{}]", proto, e); - } - } +// @Override +// public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { +// try { +// TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); +// +// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); +// if (tpi.isMyPartition()) { +// log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); +// if (proto.getDeleted()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// } +// if (proto.getAdded()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// if (proto.getUpdated()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// } else { +// clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); +// } +// } catch (Exception e) { +// log.trace("Failed to process entity update msg: [{}]", proto, e); +// } +// } private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..d37e7ebc3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -28,6 +28,6 @@ public interface CalculatedFieldStateService { void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void removeState(CalculatedFieldEntityCtxId ctxId); + void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 37e402fbc8..6ca4369230 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -154,4 +153,12 @@ public class CalculatedFieldCtx { public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } + + public boolean hasSignificantChanges(CalculatedFieldCtx other) { + boolean entityIdChanged = !entityId.equals(other.entityId); + boolean typeChanged = !cfType.equals(other.cfType); + boolean argumentsChanged = !arguments.equals(other.arguments); + return entityIdChanged || typeChanged || argumentsChanged; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..f5c882ea52 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -59,8 +58,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId) { + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + callback.onSuccess(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 95b340c362..d40bf7b130 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.Set; @@ -124,7 +125,7 @@ public class EntityStateSourcingListener { tbClusterService.onApiStateChange(apiUsageState, null); } case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); } default -> { } @@ -193,8 +194,7 @@ public class EntityStateSourcingListener { } case CALCULATED_FIELD -> { CalculatedField calculatedField = (CalculatedField) event.getEntity(); - ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); - tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + tbClusterService.onCalculatedFieldDeleted(calculatedField, null); } default -> { } @@ -276,13 +276,13 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { CalculatedField calculatedField = (CalculatedField) entity; CalculatedField oldCalculatedField = null; if (oldEntity instanceof CalculatedField) { oldCalculatedField = (CalculatedField) oldEntity; } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); } private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0fcc04bd5d..3d8bbddc80 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -25,20 +24,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -70,6 +66,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.thingsboard.server.common.util.ProtoUtils.fromProto; + @Service @TbRuleEngineComponent @Slf4j @@ -86,8 +84,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private final TbRuleEngineQueueFactory queueFactory; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; private volatile ListeningExecutorService calculatedFieldsExecutor; @@ -101,12 +97,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldExecutionService calculatedFieldExecutionService, CalculatedFieldCache calculatedFieldCache) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -168,12 +162,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } else if (toCfMsg.hasComponentLifecycleMsg()) { log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); - } else if (toCfMsg.hasEntityUpdateMsg()) { - log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -222,14 +211,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); - if (toCfNotification.hasComponentLifecycle()) { + if (toCfNotification.hasComponentLifecycleMsg()) { // from upstream (maybe removed since we don't need to init state for each partition) - forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); - handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); - } else if (toCfNotification.hasEntityUpdateMsg()) { - processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we don't need to update state for each partition) - forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); + forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); } else if (toCfNotification.hasLinkedTelemetryMsg()) { forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } @@ -248,74 +233,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { - var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); - if (entityUpdateMsg.getAdded()) { - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } else if (entityUpdateMsg.getDeleted()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - } else if (entityUpdateMsg.getUpdated()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } - } - - private void throwNotHandled(Object msg, TbCallback callback) { - log.warn("Message not handled: {}", msg); - callback.onFailure(new RuntimeException("Message not handled!")); + private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { + var msg = fromProto(proto); + actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); } private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2783e826fe..b272694b16 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -70,7 +70,6 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -362,8 +361,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toRuleEngineMsgs.incrementAndGet(); + pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @Override @@ -435,7 +433,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, deviceId); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -444,7 +442,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, assetId); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -597,9 +595,7 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -622,14 +618,6 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } - if (entityType.equals(EntityType.CALCULATED_FIELD)) { - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -669,56 +657,86 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceUpdated(Device device, Device old) { + public void onDeviceUpdated(Device entity, Device old) { var created = old == null; - broadcastEntityChangeToTransport(device.getTenantId(), device.getId(), device, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean deviceNameChanged = !device.getName().equals(old.getName()); + boolean deviceNameChanged = !entity.getName().equals(old.getName()); if (deviceNameChanged) { - gatewayNotificationsService.onDeviceUpdated(device, old); + gatewayNotificationsService.onDeviceUpdated(entity, old); } - boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + boolean deviceProfileChanged = !entity.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getDeviceProfileId()) + .profileId(entity.getDeviceProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { - pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); + pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); } } else { - handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getDeviceProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); - otaPackageStateService.update(device, old); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); + otaPackageStateService.update(entity, old); } @Override - public void onAssetUpdated(Asset asset, Asset old) { + public void onAssetUpdated(Asset entity, Asset old) { var created = old == null; - broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean assetTypeChanged = !asset.getType().equals(old.getType()); + boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); if (assetTypeChanged) { - handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getAssetProfileId()) + .profileId(entity.getAssetProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { - handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getAssetProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { - var created = oldCalculatedField == null; - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override @@ -849,54 +867,8 @@ public class DefaultTbClusterService implements TbClusterService { } } - private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - - private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); - } - - private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); - } - - private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (oldProfileId != null) { - builder.setEntityProfileType(oldProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - } - if (newProfileId != null) { - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - } - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - - broadcastEntityUpdateEvent(msg); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); - } - - private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); - Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } - } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 588b135891..3d81de1cc4 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -131,8 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index 7943c9d43d..9e8f8e4a0a 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,8 +15,22 @@ */ package org.thingsboard.server.queue; + public interface TbQueueCallback { + TbQueueCallback EMPTY = new TbQueueCallback() { + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 91419bee9c..e13e2d2ecb 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -135,14 +135,18 @@ public enum MsgType { EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; - CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; - CF_TELEMETRY_MSG, + CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_TELEMETRY_MSG, // Sent from queue to actor system; + CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; + + /* CF Manager Actor -> CF Entity actor */ CF_ENTITY_TELEMETRY_MSG, - CF_LINKED_TELEMETRY_MSG, - CF_UPDATE_MSG, - CF_ENTITY_UPDATE_MSG; + CF_ENTITY_INIT_CF_MSG, + CF_ENTITY_DELETE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java new file mode 100644 index 0000000000..51eea1f943 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityLifecycleMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final ComponentLifecycleMsg data; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_LIFECYCLE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 000733cfd8..208b6ab2f5 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,25 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; + private final String oldName; + private final String name; + private final EntityId oldProfileId; + private final EntityId profileId; + + public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { + this(tenantId, entityId, event, null, null, null, null); + } + + @Builder + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.event = event; + this.oldName = oldName; + this.name = name; + this.oldProfileId = oldProfileId; + this.profileId = profileId; + } public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..f0a34de08e 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -114,14 +114,28 @@ public class ProtoUtils { } public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { - return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) - .build(); + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())); + if (msg.getProfileId() != null) { + builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); + } + if (msg.getOldProfileId() != null) { + builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + } + if (msg.getName() != null) { + builder.setName(msg.getName()); + } + if (msg.getOldName() != null) { + builder.setName(msg.getOldName()); + } + return builder.build(); } public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { @@ -129,11 +143,22 @@ public class ProtoUtils { } public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { - return new ComponentLifecycleMsg( - TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), - ComponentLifecycleEvent.values()[proto.getEventValue()] - ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + var builder = ComponentLifecycleMsg.builder() + .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) + .entityId(entityId) + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]) + .name(proto.getName()) + .oldName(proto.getOldName()); + if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); + } + if (proto.getOldProfileIdMSB() != 0 || proto.getOldProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); + } + return builder.build(); } public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..18bcb7c6f4 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -773,22 +773,6 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldEntityUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 oldProfileIdMSB = 7; - int64 oldProfileIdLSB = 8; - int64 newProfileIdMSB = 9; - int64 newProfileIdLSB = 10; - bool added = 11; - bool updated = 12; - bool deleted = 13; -} - message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1203,6 +1187,13 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; + //Since 4.0. To replace the + string oldName = 7; + string name = 8; + int64 oldProfileIdMSB = 9; + int64 oldProfileIdLSB = 10; + int64 profileIdMSB = 11; + int64 profileIdLSB = 12; } message EdgeEventMsgProto { @@ -1630,16 +1621,14 @@ message ToEdgeEventNotificationMsg { } message ToCalculatedFieldMsg { - CalculatedFieldTelemetryMsgProto telemetryMsg = 1; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; - ComponentLifecycleMsgProto componentLifecycleMsg = 3; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 4; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldTelemetryMsgProto telemetryMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } message ToCalculatedFieldNotificationMsg { - ComponentLifecycleMsgProto componentLifecycle = 1; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From bc835eb9d467f3bcff9777e86944171cdf64a852 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 15:33:30 +0200 Subject: [PATCH 107/281] Fixes --- .../calculated-fields-table-config.ts | 12 +- ...culated-field-arguments-table.component.ts | 2 +- .../calculated-field-dialog.component.ts | 23 +++- ...ulated-field-argument-panel.component.html | 122 +++++++++--------- ...ulated-field-argument-panel.component.scss | 4 + ...lculated-field-argument-panel.component.ts | 11 +- .../entity/entity-autocomplete.component.html | 2 +- .../entity-key-autocomplete.component.html | 12 +- 8 files changed, 111 insertions(+), 77 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ae025a13f6..fcef11799d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -77,12 +77,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('name', 'common.name', '33%')); - this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); - this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '33%', entity => entity.configuration?.expression); + expressionColumn.sortable = false; + + this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push(new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push(expressionColumn); this.cellActionDescriptors.push( { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 1dedc5e80e..657b20abb4 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -133,7 +133,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }; this.keysPopupClosed = false; const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 7f9af5cead..71fea36959 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -84,10 +84,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); - this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + this.observeTypeChanges(); } get configFormGroup(): FormGroup { @@ -113,10 +110,26 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); + this.fieldFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); } private toggleScopeByOutputType(type: OutputType): void { this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); } + + private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { + this.outputFormGroup.get('name')[type === CalculatedFieldType.SIMPLE? 'enable' : 'disable']({emitEvent: false}); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 865b71406d..58923e0029 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -24,33 +24,35 @@
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - +
+ @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } - } +
@@ -117,40 +119,42 @@ } - @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { -
-
{{ 'calculated-fields.timeseries-key' | translate }}
- -
- } @else { -
-
{{ 'calculated-fields.attribute-scope' | translate }}
- - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if ((keyEntityType$ | async) === EntityType.DEVICE) { - - {{ 'calculated-fields.client-attributes' | translate }} - - - {{ 'calculated-fields.shared-attributes' | translate }} + @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} - } - - -
-
-
{{ 'calculated-fields.attribute-key' | translate }}
- -
+ @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index a784909b92..520f2d3ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -25,5 +25,9 @@ width: 100%; } } + + .mat-mdc-form-field-infix { + display: flex; + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b476bc0fee..79f34d7c20 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -92,6 +92,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme constructor( private fb: FormBuilder, + private cd: ChangeDetectorRef ) { super(); @@ -154,22 +155,24 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme default: entityId = this.argumentFormGroup.get('refEntityId').value as any; } - if (onInit) { + if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } this.entityFilter = { type: AliasFilterType.singleEntity, singleEntity: entityId, }; + this.cd.markForCheck(); } private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), this.refEntityKeyFormGroup.get('scope').valueChanges, ) - .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .pipe(throttleTime(100), delay(50), takeUntilDestroyed()) .subscribe(() => this.updateEntityFilter(this.entityType)); } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index 44a218b455..cbec73e64d 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -29,7 +29,7 @@ {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} - - close + } @else if (keyControl.hasError('required') && keyControl.touched) { + + warning + } @for (key of filteredKeys$ | async; track key) { + } @empty { + {{ 'entity.no-keys-found' | translate }} } From 75369604a1cc1410b9af6249a3eb448d162a416a Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:46:28 +0200 Subject: [PATCH 108/281] Fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../calculated-field-arguments-table.component.ts | 12 ++++++++---- .../dialog/calculated-field-dialog.component.ts | 4 +++- .../calculated-field-argument-panel.component.html | 3 ++- .../calculated-field-argument-panel.component.ts | 13 ++++--------- ui-ngx/src/app/shared/models/regex.constants.ts | 2 ++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 021bd15c58..f6b82dd954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -99,7 +99,7 @@ {{ 'calculated-fields.no-arguments' | translate }} } - @if (errorText) { + @if (errorText && this.argumentsFormArray.dirty) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 657b20abb4..7357912bdc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -98,7 +98,11 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + effect(() => { + if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { + this.argumentsFormArray.updateValueAndValidity(); + } + }); } registerOnChange(fn: (argumentsObj: Record) => void): void { @@ -183,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -202,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 71fea36959..d30a90e954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -110,7 +110,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent {{ 'calculated-fields.server-attributes' | translate }} - @if ((keyEntityType$ | async) === EntityType.DEVICE) { + @if (entityType === ArgumentEntityType.Device + || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.client-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 79f34d7c20..2949ffdaee 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -57,7 +57,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -74,11 +74,6 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentTypes: ArgumentType[]; entityFilter: EntityFilter; - keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges - .pipe( - startWith(this.refEntityIdFormGroup.get('entityType').value), - map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) - ); readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; @@ -140,7 +135,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); } - private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { let entityId: EntityId; switch (entityType) { case ArgumentEntityType.Current: diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 55742cefd4..60bf154423 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -15,3 +15,5 @@ /// export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; + +export const charNumRegex = /^[a-zA-Z0-9]+$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index be31b4f70b..ab696e226c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1030,7 +1030,7 @@ "timeseries-key": "Time series key", "device-name": "Device name", "latest-telemetry": "Latest telemetry", - "rolling": "Rolling", + "rolling": "Time series rolling", "attribute-scope": "Attribute scope", "server-attributes": "Server attributes", "client-attributes": "Client attributes", From ff8a9309f4a0cba877aa5d401a325021ed1d2cf6 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:57:46 +0200 Subject: [PATCH 109/281] Fixes --- .../calculated-field-arguments-table.component.html | 4 ++-- .../panel/calculated-field-argument-panel.component.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index f6b82dd954..4f1d1d3980 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -28,7 +28,7 @@ - @if (group.get('refEntityId')?.get('id').value) { + @if (group.get('refEntityId')?.get('id')?.value) { @@ -51,7 +51,7 @@ {{ - (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant ? 'calculated-fields.argument-current-tenant' : 'calculated-fields.argument-current') | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 2949ffdaee..6e232bdb3d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -119,7 +119,9 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme } saveArgument(): void { - this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + const { refEntityId, ...restConfig } = this.argumentFormGroup.value; + const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; + this.argumentsDataApplied.emit({ value, index: this.index }); } cancel(): void { @@ -148,7 +150,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as any; + entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); From 6905cb530b915898cf90201bde255a3507c20293 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 17:12:02 +0200 Subject: [PATCH 110/281] added new protos for states --- ...efaultCalculatedFieldExecutionService.java | 23 +-- .../server/service/cf/RocksDBService.java | 28 +++- .../cf/ctx/CalculatedFieldStateService.java | 2 - .../cf/ctx/state/RocksDBStateService.java | 149 ++++++++++++++++-- .../ctx/state/ScriptCalculatedFieldState.java | 3 +- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 28 +++- .../ctx/state/TsRollingArgumentEntryTest.java | 25 +-- .../server/common/util/ProtoUtils.java | 39 ++++- common/proto/src/main/proto/queue.proto | 25 ++- 11 files changed, 263 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 8f948aa265..7367eb611f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,13 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -84,7 +86,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -383,7 +384,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); @@ -551,7 +552,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + .collect(Collectors.toMap(Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); // updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @@ -677,7 +678,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } @@ -710,7 +711,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); builder.setMsg(telemetryProto); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -719,8 +720,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } //TODO: IM: move to utils; - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) .setEntityType(ctxId.entityId().getEntityType().name()) @@ -890,8 +893,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) .build(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 3aed65eced..5eaeff120a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -22,6 +22,8 @@ import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; @@ -49,6 +51,14 @@ public class RocksDBService { } } + public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { + try { + db.put(writeOptions, key.toByteArray(), value.toByteArray()); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + public void delete(String key) { try { db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); @@ -67,18 +77,20 @@ public class RocksDBService { } } - public Map getAll() { - Map map = new HashMap<>(); + public Map getAll() { + Map results = new HashMap<>(); try (RocksIterator iterator = db.newIterator()) { for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - String key = new String(iterator.key(), StandardCharsets.UTF_8); - String value = new String(iterator.value(), StandardCharsets.UTF_8); - map.put(key, value); + try { + CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); + CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); + results.put(key, value); + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } } - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); } - return map; + return results; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..4f9a998937 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -24,8 +24,6 @@ public interface CalculatedFieldStateService { Map restoreStates(); - CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..4de15f70bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,14 +19,28 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RollingArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueProto; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import java.util.Map; -import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; import java.util.stream.Collectors; @Service @@ -40,21 +54,14 @@ public class RocksDBStateService implements CalculatedFieldStateService { public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( - entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) + entry -> fromProto(entry.getKey()), + entry -> fromProto(entry.getValue()) )); } @Override - public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { - return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) - .orElse(null); - } - - @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ - rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); callback.onSuccess(); } @@ -63,4 +70,120 @@ public class RocksDBStateService implements CalculatedFieldStateService { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()) + .addAllRequiredArguments(state.getRequiredArguments()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + + return builder.build(); + } + + private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(entry.getTs()); + + if (entry.getVersion() != null) { + singleValueProtoBuilder.setVersion(entry.getVersion()); + } + + KvEntry value = entry.getValue(); + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + return SingleValueArgumentProto.newBuilder() + .setArgName(argName) + .setValue(singleValueProtoBuilder.build()) + .build(); + } + + private RollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + RollingArgumentProto.Builder rollingArgumentProtoBuilder = RollingArgumentProto.newBuilder() + .setArgName(argName); + + entry.getTsRecords().forEach((ts, value) -> { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(ts); + + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + rollingArgumentProtoBuilder.addValues(singleValueProtoBuilder.build()); + }); + + return rollingArgumentProtoBuilder.build(); + } + + private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(proto.getRequiredArgumentsList()); + case SCRIPT -> new ScriptCalculatedFieldState(proto.getRequiredArgumentsList()); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + SingleValueProto valueProto = proto.getValue(); + BasicKvEntry value = valueProto.getHasV() ? ProtoUtils.fromProto(valueProto.getValue()) : null; + + return new SingleValueArgumentEntry(valueProto.getTs(), value, valueProto.getVersion()); + } + + private TsRollingArgumentEntry fromRollingArgumentProto(RollingArgumentProto proto) { + TreeMap tsRecords = new TreeMap<>(); + + proto.getValuesList().forEach(singleValueProto -> { + BasicKvEntry value = singleValueProto.getHasV() ? ProtoUtils.fromProto(singleValueProto.getValue()) : null; + tsRecords.put(singleValueProto.getTs(), value); + }); + + return new TsRollingArgumentEntry(tsRecords); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index a3c7efb388..4a24d13c93 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -53,7 +54,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { arguments.forEach((key, argumentEntry) -> { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = tsRollingEntry.getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 8b8fe6e8c7..d233b60512 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,6 +21,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -52,7 +53,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5117fcab0e..8d4d40d39b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.util.ProtoUtils; @@ -33,19 +34,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private Object value; + private BasicKvEntry value; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -56,7 +57,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = entry.getValue(); + this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -72,11 +73,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } - @Override - public Object getValue() { - return value; - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index b86a51ca03..6c1a772c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -21,11 +21,15 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.ProtoUtils; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; @Data @NoArgsConstructor @@ -37,10 +41,10 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; - private TreeMap tsRecords = new TreeMap<>(); + private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } /** @@ -58,7 +62,15 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords; + return tsRecords.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().getValue(), + (oldValue, newValue) -> oldValue, + TreeMap::new + )); + } @Override @@ -79,7 +91,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { boolean updated = false; - for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); } return updated; @@ -89,7 +101,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); } - private boolean addTsRecordIfAbsent(Long ts, Object value) { + private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { if (!tsRecords.containsKey(ts)) { addTsRecord(ts, value); return true; @@ -97,9 +109,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return false; } - private void addTsRecord(Long ts, Object value) { - if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(ts, value); + private void addTsRecord(Long ts, KvEntry value) { + if (NumberUtils.isParsable(value.getValue().toString())) { + tsRecords.put(ts, ProtoUtils.basicKvEntryFromKvEntry(value)); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index b08c5f2a58..9ca242092d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import java.util.Map; import java.util.TreeMap; @@ -32,10 +35,10 @@ public class TsRollingArgumentEntryTest { @BeforeEach void setUp() { - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new DoubleDataEntry("key", 10.0)); + values.put(ts - 30, new DoubleDataEntry("key", 12.0)); + values.put(ts - 20, new DoubleDataEntry("key", 17.0)); entry = new TsRollingArgumentEntry(values); } @@ -47,7 +50,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); @@ -56,7 +59,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isFalse(); } @@ -64,10 +67,10 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16); - values.put(ts - 10, 7); - values.put(ts - 5, 1); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new DoubleDataEntry("key", 16.0)); + values.put(ts - 10, new DoubleDataEntry("key", 7.0)); + values.put(ts - 5, new DoubleDataEntry("key", 1.0)); newEntry.setTsRecords(values); assertThat(entry.updateEntry(newEntry)).isTrue(); @@ -83,7 +86,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenValueIsNotNumber() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..91e3276210 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -90,7 +91,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -630,6 +631,42 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static BasicKvEntry basicKvEntryFromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + } + + public static BasicKvEntry fromProto(KeyValueProto proto) { + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, proto.getBoolV()); + case LONG_V -> new LongDataEntry(key, proto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, proto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, proto.getStringV()); + case JSON_V -> new JsonDataEntry(key, proto.getJsonV()); + default -> null; + }; + } + + public static BasicKvEntry basicKvEntryFromKvEntry(KvEntry kvEntry) { + String key = kvEntry.getKey(); + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BooleanDataEntry(key, kvEntry.getBooleanValue().orElse(null)); + case LONG -> new LongDataEntry(key, kvEntry.getLongValue().orElse(null)); + case DOUBLE -> new DoubleDataEntry(key, kvEntry.getDoubleValue().orElse(null)); + case STRING -> new StringDataEntry(key, kvEntry.getStrValue().orElse(null)); + case JSON -> new JsonDataEntry(key, kvEntry.getJsonValue().orElse(null)); + }; + } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { TransportProtos.KeyValueProto kvProto = proto.getKv(); String key = kvProto.getKey(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 53045fc147..2ed535495a 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -810,11 +810,13 @@ message CalculatedFieldLinkedTelemetryMsgProto { } message CalculatedFieldEntityCtxIdProto { - int64 calculatedFieldIdMSB = 1; - int64 calculatedFieldIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; } message CalculatedFieldIdProto { @@ -825,13 +827,8 @@ message CalculatedFieldIdProto { message SingleValueProto { int64 ts = 1; int64 version = 2; - KeyValueType type = 3; bool has_v = 4; - bool bool_v = 5; - int64 long_v = 6; - double double_v = 7; - string string_v = 8; - string json_v = 9; + KeyValueProto value = 5; } message SingleValueArgumentProto { @@ -847,8 +844,10 @@ message RollingArgumentProto { message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; // int32 version = 2; - repeated SingleValueArgumentProto singleValueArguments = 3; - repeated RollingArgumentProto rollingValueArguments = 4; + string type = 3; + repeated string requiredArguments = 4; + repeated SingleValueArgumentProto singleValueArguments = 5; + repeated RollingArgumentProto rollingValueArguments = 6; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 14feedbfa05b11f1d20dfa0f6fe9c17a7ea03f57 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 18:01:28 +0200 Subject: [PATCH 111/281] Changed UI of timeWindow --- ...ulated-field-argument-panel.component.html | 22 +++++++++---------- ...lculated-field-argument-panel.component.ts | 5 +++-- .../assets/locale/locale.constant-en_US.json | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 7c33cc2eb3..b1d463a2d3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -168,22 +168,20 @@ } @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- - - {{ 'common.suffix.ms' | translate }} - +
+
{{ 'calculated-fields.time-window' | translate }}
+
+
{{ 'calculated-fields.limit' | translate }}
-
- - - -
+
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 6e232bdb3d..354b6b9f58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -36,6 +36,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityFilter } from '@shared/models/query/query.models'; import { AliasFilterType } from '@shared/models/alias.models'; import { merge } from 'rxjs'; +import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,8 +69,8 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [null], - timeWindow: [null], + limit: [10], + timeWindow: [MINUTE * 15], }); argumentTypes: ArgumentType[]; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ab696e226c..f0f0671c5c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1037,7 +1037,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", - "limit": "Limit", + "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", "timeseries": "Time series", From 8dca91c909138deb18a5b8d1b6aed332bcad2e2e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:08:56 +0200 Subject: [PATCH 112/281] Fixed device filter bug --- .../calculated-fields/calculated-fields-table-config.ts | 6 +++--- .../calculated-fields-table.component.html | 4 +++- .../calculated-fields/calculated-fields-table.component.ts | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index fcef11799d..58caa759da 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -97,7 +97,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.editCalculatedField(entity) + onAction: (_, entity) => this.editCalculatedField(entity), } ); } @@ -171,7 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig - +@if (calculatedFieldsTableConfig) { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4c22ad2896..4fda1cc075 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,6 +16,7 @@ import { ChangeDetectionStrategy, + ChangeDetectorRef, Component, DestroyRef, effect, @@ -43,7 +44,7 @@ export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - active = input(); + active = input(); entityId = input(); calculatedFieldsTableConfig: CalculatedFieldsTableConfig; @@ -54,6 +55,7 @@ export class CalculatedFieldsTableComponent { private store: Store, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, + private cd: ChangeDetectorRef, private destroyRef: DestroyRef) { effect(() => { @@ -68,6 +70,7 @@ export class CalculatedFieldsTableComponent { this.popoverService, this.destroyRef, ); + this.cd.markForCheck(); } }); } From dd8ce35b86f2da4102bb1bfebceada167bc72a99 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:18:57 +0200 Subject: [PATCH 113/281] Fixes --- .../calculated-field-arguments-table.component.ts | 6 +++--- .../dialog/calculated-field-dialog.component.ts | 2 +- .../panel/calculated-field-argument-panel.component.html | 2 +- .../panel/calculated-field-argument-panel.component.ts | 8 ++++---- .../entity/entity-key-autocomplete.component.html | 4 +++- ui-ngx/src/app/shared/models/regex.constants.ts | 2 +- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 7357912bdc..c8dae67aec 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { charNumRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -187,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -206,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d30a90e954..6d0687fe9a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -61,7 +61,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index b1d463a2d3..020fad4fd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -23,7 +23,7 @@
{{ 'calculated-fields.argument-name' | translate }}
- +
@if (argumentFormGroup.get('argumentName').touched) { @if (argumentFormGroup.get('argumentName').hasError('required')) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 354b6b9f58..792742e5d0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -58,7 +58,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -66,10 +66,10 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], key: [''], - scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [10], + limit: [1000], timeWindow: [MINUTE * 15], }); diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html index 3e4ff6e90d..f1073738f9 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html @@ -43,7 +43,9 @@ @for (key of filteredKeys$ | async; track key) { } @empty { - {{ 'entity.no-keys-found' | translate }} + @if (!this.keyControl.value) { + {{ 'entity.no-keys-found' | translate }} + } } diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 60bf154423..c6b231ef6e 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -16,4 +16,4 @@ export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; -export const charNumRegex = /^[a-zA-Z0-9]+$/; +export const charsWithNumRegex = /^[a-zA-Z]+[a-zA-Z0-9]*$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f0f0671c5c..ee348cb6c3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1048,7 +1048,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "hint": { - "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", "expression-required": "Expression is required.", "expression-invalid": "Expression is invalid", From c82b6c8c45291d35bb15b128d7313167dd8a9dda Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:20:26 +0200 Subject: [PATCH 114/281] Empty arguments fix --- .../components/dialog/calculated-field-dialog.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 6d0687fe9a..c9f1a22157 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -34,7 +34,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -63,6 +63,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); From 1a67769f1c3657bca7e1a8a31e558c7c0e5ac0e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:01:06 +0200 Subject: [PATCH 115/281] added api limits and fixed tests --- ...CalculatedFieldEntityMessageProcessor.java | 28 ++++++------ ...alculatedFieldManagerMessageProcessor.java | 14 ++++++ ...efaultCalculatedFieldExecutionService.java | 8 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 3 +- .../cf/ctx/state/RocksDBStateService.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++--- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../cf/DefaultTbCalculatedFieldService.java | 18 -------- .../queue/DefaultTbClusterService.java | 6 +-- .../state/ScriptCalculatedFieldStateTest.java | 43 ++++++++++--------- .../state/SimpleCalculatedFieldStateTest.java | 13 +++--- .../state/SingleValueArgumentEntryTest.java | 15 ++++--- .../ctx/state/TsRollingArgumentEntryTest.java | 12 +++--- .../DefaultTenantProfileConfiguration.java | 7 +++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../CalculatedFieldDataValidator.java | 31 +++++++++++++ .../dao/sql/cf/CalculatedFieldRepository.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 19 files changed, 145 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f0064ff459..9392e3c02a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -198,20 +198,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } - private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { - if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { - return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); - } - return null; - } - - private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTbMsgType().isEmpty()) { - return TbMsgType.valueOf(proto.getTbMsgType()); - } - return null; - } - @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { if (state.isReady()) { @@ -290,4 +276,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return cfIds; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index a2577a3f27..d5ed8b8aaa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -217,6 +217,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -257,6 +264,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } initCf(newCfCtx, callback, true); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index b9a18084b1..d8e42c916d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -61,6 +61,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -69,6 +70,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -142,14 +144,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; + private final ApiLimitService apiLimitService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap states = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; - private static final Set supportedReferencedEntities = EnumSet.of( EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT ); @@ -560,7 +561,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index caeeabea52..a56aed64f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -197,7 +197,8 @@ public class CalculatedFieldCtx { boolean entityIdChanged = !entityId.equals(other.entityId); boolean typeChanged = !cfType.equals(other.cfType); boolean argumentsChanged = !arguments.equals(other.arguments); - return entityIdChanged || typeChanged || argumentsChanged; + boolean expressionChanged = !expression.equals(other.expression); + return entityIdChanged || typeChanged || argumentsChanged || expressionChanged; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index eaa1b08f5e..bfed563182 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -115,7 +115,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { singleValueProtoBuilder.setVersion(entry.getVersion()); } - KvEntry value = entry.getValue(); + KvEntry value = entry.getKvEntryValue(); if (value != null) { singleValueProtoBuilder.setHasV(true) .setValue(ProtoUtils.toKeyValueProto(value)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d233b60512..01cf9cff70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -53,7 +53,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d4d40d39b..0832e53e5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -34,19 +35,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private BasicKvEntry value; + private BasicKvEntry kvEntryValue; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry.getKv()); + this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.basicKvEntryFromProto(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -57,7 +58,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -65,7 +66,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { * */ private SingleValueArgumentEntry(int ignored) { this.ts = System.currentTimeMillis(); - this.value = null; + this.kvEntryValue = null; } @Override @@ -73,9 +74,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } + @JsonIgnore + public Object getValue() { + return kvEntryValue.getValue(); + } + @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value, this.version); + return new SingleValueArgumentEntry(this.ts, this.kvEntryValue, this.version); } @Override @@ -88,7 +94,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.value = singleValueEntry.getValue(); + this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 6c1a772c44..ddae0c3513 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -98,7 +98,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { - return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); } private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 184092720c..1ccfe2375c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -48,9 +48,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private static final int MAX_ARGUMENT_SIZE = 10; - private static final int MAX_CALCULATED_FIELD_NUMBER = 10; - private final CalculatedFieldService calculatedFieldService; @Override @@ -62,9 +59,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); checkForEntityChange(existingCf, calculatedField); } - checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); - checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -129,19 +124,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } - private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { - if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { - throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); - } - } - - private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { - int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); - if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { - throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); - } - } - private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b272694b16..944e24480e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -103,7 +102,6 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -729,13 +727,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index cbaea6575c..42ff828dbd 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +53,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -65,6 +67,7 @@ public class ScriptCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -93,7 +96,7 @@ public class ScriptCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -116,17 +119,17 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4);// will not be used for calculation - values.put(ts - 45000, 2);// will not be used for calculation - values.put(ts - 20, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, new LongDataEntry("deviceTemperature", 4L));// will not be used for calculation + values.put(ts - 45000, new LongDataEntry("deviceTemperature", 2L));// will not be used for calculation + values.put(ts - 20, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); @@ -138,19 +141,19 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000);// will not be used - values.put(ts - 18, 0); - values.put(ts - 16, 0); - values.put(ts - 14, 0); - values.put(ts - 12, 0); - values.put(ts - 10, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 1000L));// will not be used + values.put(ts - 18, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 16, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 14, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 12, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 10, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); @@ -161,7 +164,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test @@ -187,10 +190,10 @@ public class ScriptCalculatedFieldStateTest { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); long ts = System.currentTimeMillis(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new LongDataEntry("deviceTemperature", 10L)); + values.put(ts - 30, new LongDataEntry("deviceTemperature", 12L)); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 17L)); argumentEntry.setTsRecords(values); return argumentEntry; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 58a981824c..d6b384d85b 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -46,9 +48,9 @@ public class SimpleCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); - private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); - private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; @@ -56,6 +58,7 @@ public class SimpleCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), null); + ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } @@ -88,7 +91,7 @@ public class SimpleCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -130,7 +133,7 @@ public class SimpleCalculatedFieldStateTest { void testPerformCalculationWhenPassedNotNumber() { state.arguments = new HashMap<>(Map.of( "key1", key1ArgEntry, - "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), "key3", key3ArgEntry )); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 285da0b423..203d7b3d71 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,7 +30,7 @@ public class SingleValueArgumentEntryTest { @BeforeEach void setUp() { - entry = new SingleValueArgumentEntry(ts, 11, 363L); + entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); } @Test @@ -46,26 +47,26 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWithThaSameTs() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } @Test void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); - assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13L); assertThat(entry.getVersion()).isNull(); } @Test void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); - assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18L); assertThat(entry.getVersion()).isEqualTo(369L); } @Test void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index 9ca242092d..5a6f5e96e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -54,7 +54,7 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); - assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + assertThat(entry.getTsRecords().get(ts - 10).getValue()).isEqualTo(23.0); } @Test @@ -76,11 +76,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, 10, - ts - 30, 12, - ts - 20, 17, - ts - 10, 7, - ts - 5, 1 + ts - 40, new DoubleDataEntry("key", 10.0), + ts - 30, new DoubleDataEntry("key", 12.0), + ts - 20, new DoubleDataEntry("key", 17.0), + ts - 10, new DoubleDataEntry("key", 7.0), + ts - 5, new DoubleDataEntry("key", 1.0) )); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 6aa9075a79..0fd630fead 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,6 +135,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; + private long maxCalculatedFieldsPerTenant; + private long maxCalculatedFieldsPerEntity; + private long maxArgumentsPerCF; + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + @Override public long getProfileThreshold(ApiUsageRecordKey key) { return switch (key) { @@ -175,6 +181,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura case DASHBOARD -> maxDashboards; case RULE_CHAIN -> maxRuleChains; case EDGE -> maxEdges; + case CALCULATED_FIELD -> maxCalculatedFieldsPerTenant; default -> 0; }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 23a2eae93e..3efb4011ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -43,4 +43,6 @@ public interface CalculatedFieldDao extends Dao { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + long countCFByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 80e421f350..db8997ceb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,11 +17,15 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; @Component public class CalculatedFieldDataValidator extends DataValidator { @@ -29,13 +33,40 @@ public class CalculatedFieldDataValidator extends DataValidator @Autowired private CalculatedFieldDao calculatedFieldDao; + @Autowired + private ApiLimitService apiLimitService; + + @Override + protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { + validateNumberOfEntitiesPerTenant(tenantId, EntityType.CALCULATED_FIELD); + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + } + @Override protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing calculated field!"); } + validateNumberOfArgumentsPerCF(tenantId, calculatedField); return old; } + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + long countCFByEntityId = calculatedFieldDao.countCFByEntityId(tenantId, entityId); + + if (countCFByEntityId == maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { + long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { + throw new DataValidationException("Calculated field arguments limit reached!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index c0118d4f02..2aeca659bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -38,4 +38,6 @@ public interface CalculatedFieldRepository extends JpaRepository removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 34bfa27b16..703bdfbf6f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -88,6 +88,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; From e62e7fb20be59db8de4cfc7a61fa2c2a6e44ea56 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:26:32 +0200 Subject: [PATCH 116/281] implemented linkMatches() method and deletion of state from db --- .../CalculatedFieldManagerMessageProcessor.java | 2 +- .../server/service/cf/RocksDBService.java | 4 ++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 15 +++++++++++++-- .../service/cf/ctx/state/RocksDBStateService.java | 3 +-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d5ed8b8aaa..b2ff883236 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -122,7 +122,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedFields.containsKey(msg.getId().cfId())) { getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - // TODO: remove state from storage + cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 5eaeff120a..fe800f61ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -59,9 +59,9 @@ public class RocksDBService { } } - public void delete(String key) { + public void delete(CalculatedFieldEntityCtxIdProto key) { try { - db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + db.delete(writeOptions, key.toByteArray()); } catch (RocksDBException e) { log.error("Failed to delete data from RocksDB", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a56aed64f7..4600628838 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -185,8 +186,18 @@ public class CalculatedFieldCtx { } public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { - //TODO: IM - implement - return true; + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry, scope); + } } public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index bfed563182..6487ce1a43 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -67,7 +66,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(toProto(ctxId)); callback.onSuccess(); } From 0590b5b271c0f464f8ce7ec75f22c669b2ebb979 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 3 Feb 2025 17:54:11 +0200 Subject: [PATCH 117/281] Improvements, refactoring, review comments resolve --- .../calculated-fields-table-config.ts | 36 ++-- .../calculated-fields-table.component.ts | 3 + ...lated-field-arguments-table.component.html | 184 +++++++++--------- ...lated-field-arguments-table.component.scss | 13 +- ...culated-field-arguments-table.component.ts | 44 ++--- .../calculated-field-dialog.component.html | 15 +- .../calculated-field-dialog.component.ts | 32 +-- ...ulated-field-argument-panel.component.html | 131 +++++-------- ...ulated-field-argument-panel.component.scss | 33 ---- ...lculated-field-argument-panel.component.ts | 22 +-- .../entity-debug-settings-button.component.ts | 1 - .../entity-debug-settings-panel.component.ts | 6 +- .../entity/entities-table.component.ts | 4 +- .../entity/entity-table-component.models.ts | 1 - .../pages/device/device-tabs.component.html | 3 +- .../entity/entity-autocomplete.component.html | 12 +- .../entity/entity-autocomplete.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 22 ++- .../shared/models/calculated-field.models.ts | 14 +- ui-ngx/src/app/shared/models/constants.ts | 1 + .../assets/locale/locale.constant-en_US.json | 4 + 21 files changed, 265 insertions(+), 320 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 58caa759da..d8c02558f8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -19,7 +19,7 @@ import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.m import { TranslateService } from '@ngx-translate/core'; import { Direction } from '@shared/models/page/sort-order'; import { MatDialog } from '@angular/material/dialog'; -import { TimePageLink } from '@shared/models/page/page-link'; +import { PageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -27,7 +27,7 @@ import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; -import { DestroyRef } from '@angular/core'; +import { DestroyRef, Renderer2 } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -35,10 +35,10 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDialogComponent } from './components/public-api'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = @@ -54,20 +54,16 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: PageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); @@ -102,12 +98,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: PageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { - const { renderer, viewContainerRef } = this.getTable(); + const { viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); } @@ -115,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.onDebugConfigChanged(id.id, settings); debugStrategyPopover.hide(); @@ -133,17 +128,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + return this.getCalculatedFieldDialog() .pipe( filter(Boolean), switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) - .subscribe((res) => { - if (res) { - this.updateData(); - } - }); } private editCalculatedField(calculatedField: CalculatedField): void { @@ -159,8 +149,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { - return this.dialog.open(CalculatedFieldDialogComponent, { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add'): Observable { + return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4fda1cc075..bc979a5f0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -21,6 +21,7 @@ import { DestroyRef, effect, input, + Renderer2, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -56,6 +57,7 @@ export class CalculatedFieldsTableComponent { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private cd: ChangeDetectorRef, + private renderer: Renderer2, private destroyRef: DestroyRef) { effect(() => { @@ -69,6 +71,7 @@ export class CalculatedFieldsTableComponent { this.durationLeft, this.popoverService, this.destroyRef, + this.renderer ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4f1d1d3980..d1d9998e5a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -15,96 +15,100 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
-
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id')?.value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ - (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate - }} - - - - } - - - @if (group.get('refEntityKey').get('type').value; as type) { - - - {{ ArgumentTypeTranslations.get(type) | translate }} - - +
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + +
+ @if (group.get('refEntityId')?.get('id')?.value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } +
+ + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} } - - - -
- {{ group.get('refEntityKey').get('key').value }} -
- - - -
- -
-
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
- @if (errorText && this.argumentsFormArray.dirty) { - - } -
-
- + @if (errorText && this.argumentsFormArray.dirty) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8695ee4068..73f03dc497 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,18 +14,7 @@ * limitations under the License. */ :host ::ng-deep { - .inline-entity-autocomplete { - .mat-mdc-form-field-infix { - padding-top: 8px; - padding-bottom: 8px; - min-height: 40px; - width: auto; - .mdc-text-field__input, .mat-mdc-select { - font-weight: 400; - line-height: 20px; - } - } - + .tb-inline-field { a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index c8dae67aec..328a82184b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,7 +17,6 @@ import { ChangeDetectorRef, Component, - DestroyRef, effect, forwardRef, input, @@ -29,6 +28,7 @@ import { AbstractControl, ControlValueAccessor, FormBuilder, + FormGroup, NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, @@ -51,6 +51,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -78,20 +79,19 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces errorText = ''; argumentsFormArray = this.fb.array([]); - keysPopupClosed = true; readonly entityTypeTranslations = entityTypeTranslations; readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, private popoverService: TbPopoverService, private viewContainerRef: ViewContainerRef, - private destroyRef: DestroyRef, private cd: ChangeDetectorRef, private renderer: Renderer2 ) { @@ -123,6 +123,9 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } const trigger = matButton._elementRef.nativeElement; if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); @@ -135,27 +138,22 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, }; - this.keysPopupClosed = false; - const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); - argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; - argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { - argumentsPanelPopover.hide(); + this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + this.popoverComponent.hide(); const formGroup = this.getArgumentFormGroup(value); if (isDefinedAndNotNull(index)) { this.argumentsFormArray.setControl(index, formGroup); } else { this.argumentsFormArray.push(formGroup); } - this.argumentsFormArray.markAsDirty(); + formGroup.markAsDirty(); this.cd.markForCheck(); }); - argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { - this.keysPopupClosed = true; - }); } } @@ -171,8 +169,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); + return this.argumentsFormArray.getRawValue().reduce((acc, rawValue) => { const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; acc[argumentName] = argument; return acc; @@ -186,24 +183,15 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], + const value: CalculatedFieldArgumentValue = { ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); + argumentName: key + }; + this.argumentsFormArray.push(this.getArgumentFormGroup(value), {emitEvent: false}); }); } - private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): FormGroup { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index e6adc1b4d8..ca27ac6fd1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -19,7 +19,7 @@

{{ 'entity.type-calculated-field' | translate}}

-
+
{{ 'common.type' | translate }} - + @for (type of fieldTypes; track type) { {{ CalculatedFieldTypeTranslations.get(type) | translate}} } @@ -102,10 +103,10 @@
{{ 'calculated-fields.output' | translate }}
-
+
{{ 'calculated-fields.output-type' | translate }} - + @for (type of outputTypes; track type) { {{ OutputTypeTranslations.get(type) | translate}} } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c9f1a22157..c8b2073309 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -18,10 +18,9 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { helpBaseUrl } from '@shared/models/constants'; import { CalculatedField, CalculatedFieldConfiguration, @@ -35,7 +34,6 @@ import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; -import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -50,7 +48,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + map(configuration => Object.keys(configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; @@ -73,7 +71,6 @@ export class CalculatedFieldDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, - public dialogRef: MatDialogRef, - public fb: UntypedFormBuilder) { + protected dialogRef: MatDialogRef, + private fb: FormBuilder) { super(store, router, dialogRef); this.applyDialogData(); this.observeTypeChanges(); @@ -104,14 +101,15 @@ export class CalculatedFieldDialogComponent extends DialogComponent
{{ 'calculated-fields.argument-name' | translate }}
-
- - -
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - - } - } -
- -
+ + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } +
-
{{ 'entity.entity-type' | translate }}
+
{{ 'entity.entity-type' | translate }}
@for (type of argumentEntityTypes; track type) { @@ -67,34 +61,17 @@
- @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { -
-
{{ 'calculated-fields.device-name' | translate }}
- -
- } @else if (entityType === ArgumentEntityType.Customer) { + @if (ArgumentEntityTypeParamsMap.has(entityType)) {
-
{{ 'calculated-fields.customer-name' | translate }}
+
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
} @@ -123,13 +100,13 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
{{ 'calculated-fields.attribute-scope' | translate }}
- - + + {{ 'calculated-fields.server-attributes' | translate }} @@ -149,7 +126,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
{{ 'calculated-fields.default-value' | translate }}
-
- - - -
+ + +
} @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- -
+
+
{{ 'calculated-fields.time-window' | translate }}
+
{{ 'calculated-fields.limit' | translate }}
- +
}
-
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 73f03dc497..8f8b1d5d81 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,4 +19,9 @@ font-size: 14px; } } + .edit-hint { + .mat-mdc-form-field-error { + font-size: 16px; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 328a82184b..a004e3db6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -21,7 +21,9 @@ import { forwardRef, input, Input, + OnChanges, Renderer2, + SimpleChanges, ViewContainerRef, } from '@angular/core'; import { @@ -70,10 +72,11 @@ import { TbPopoverComponent } from '@shared/components/popover.component'; } ], }) -export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges { @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; calculatedFieldType = input() @@ -84,6 +87,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; @@ -105,6 +110,13 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }); } + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.markAsDirty(); + } + } + registerOnChange(fn: (argumentsObj: Record) => void): void { this.propagateChange = fn; } @@ -137,6 +149,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces calculatedFieldType: this.calculatedFieldType(), buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, + entityName: this.entityName, }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -202,6 +215,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), } : {}), refEntityKey: this.fb.group({ + ...value.refEntityKey, type: [{ value: value.refEntityKey.type, disabled: true }], key: [{ value: value.refEntityKey.key, disabled: true }], }), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ca27ac6fd1..37e96cfd91 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -69,6 +69,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" + [entityName]="data.entityName" [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -114,7 +115,7 @@ @if (outputFormGroup.get('type').value === OutputType.Attribute) { - {{ 'calculated-fields.output-type' | translate }} + {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c8b2073309..155a384273 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,6 +36,7 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -59,10 +60,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(configuration.arguments)) + startWith(null), + map(() => Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 56568a7bfe..d274c30b8f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -96,7 +96,7 @@ }
- @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 60d79d7bd6..b47d2f073c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -49,6 +49,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() argument: CalculatedFieldArgumentValue; @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -83,6 +84,8 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { readonly ArgumentEntityType = ArgumentEntityType; readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + private currentEntityFilter: EntityFilter; + constructor( private fb: FormBuilder, private cd: ChangeDetectorRef, @@ -107,6 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = this.getCurrentEntityFilter(); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -138,30 +142,53 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { } private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { - let entityId: EntityId; + let entityFilter: EntityFilter; switch (entityType) { case ArgumentEntityType.Current: - entityId = this.entityId + entityFilter = this.currentEntityFilter; break; case ArgumentEntityType.Tenant: - entityId = { - id: this.tenantId, - entityType: EntityType.TENANT + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } - this.entityFilter = { - type: AliasFilterType.singleEntity, - singleEntity: entityId, - }; + this.entityFilter = entityFilter; this.cd.markForCheck(); } + private getCurrentEntityFilter(): EntityFilter { + switch (this.entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: this.entityId, + }; + } + } + private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index 9084301783..f70d4f443b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 4329cd7e3c..2b677dc278 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index aa928687ad..37cfafa9aa 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -69,6 +69,10 @@
+ + + diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 533384e1c4..f73926f62b 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -120,6 +120,7 @@ export interface CalculatedFieldDialogData { entityId: EntityId; debugLimitsConfiguration: string; tenantId: string; + entityName?: string; } export interface ArgumentEntityTypeParams { From d14d0d4e8a8f011df869fedd96c9a09964c60af4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Feb 2025 17:03:25 +0200 Subject: [PATCH 122/281] used limits for state persistence --- .../server/actors/ActorSystemContext.java | 6 ++++++ .../CalculatedFieldEntityMessageProcessor.java | 4 ++-- .../CalculatedFieldManagerMessageProcessor.java | 6 +++--- .../service/cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/DefaultCalculatedFieldCache.java | 4 +++- .../cf/DefaultCalculatedFieldExecutionService.java | 4 ++-- .../service/cf/ctx/CalculatedFieldStateService.java | 3 ++- .../cf/ctx/state/BaseCalculatedFieldState.java | 5 +++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 10 +++++++++- .../service/cf/ctx/state/RocksDBStateService.java | 7 +++++-- .../ctx/state/ScriptCalculatedFieldStateTest.java | 12 ++++++++++-- .../ctx/state/SimpleCalculatedFieldStateTest.java | 13 ++++++++++++- .../common/data/cf/configuration/Argument.java | 6 ++++-- .../server/common/data/cf/configuration/Output.java | 2 ++ .../data/cf/configuration/ReferencedEntityKey.java | 4 ++-- 15 files changed, 66 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..49e2c9c46e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -98,6 +98,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9392e3c02a..f6bc5b2d63 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -200,7 +200,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { @@ -209,7 +209,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index ec86e640d4..31c8eee23c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -95,7 +95,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -220,7 +220,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -248,7 +248,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getId()); List newCfList = new ArrayList<>(oldCfList.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 19f60165cf..393fbd3ec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,7 +43,7 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..c8f9a7e882 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -57,6 +58,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final AssetService assetService; private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ApiLimitService apiLimitService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -116,7 +118,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { if (ctx == null) { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { - ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..03c8f3de0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -265,8 +265,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(stateId, state, callback); + public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(ctx, stateId, state, callback); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index c670e97580..e822d52767 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Map; @@ -24,7 +25,7 @@ public interface CalculatedFieldStateService { Map restoreStates(); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 21105a44cb..86d83a1f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import java.util.ArrayList; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; @@ -34,8 +36,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } public BaseCalculatedFieldState() { - this.requiredArguments = new ArrayList<>(); - this.arguments = new HashMap<>(); + this(new ArrayList<>(), new HashMap<>()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 4600628838..c483c6ab5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -67,7 +69,10 @@ public class CalculatedFieldCtx { private boolean initialized; - public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { this.calculatedField = calculatedField; this.cfId = calculatedField.getId(); @@ -96,6 +101,9 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSizeInKBytes = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes); } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 18e604118d..371751cc6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -59,8 +59,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateProto stateProto = toProto(stateId, state); + if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); + } callback.onSuccess(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 42ff828dbd..bded0d058d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -45,6 +47,8 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; @SpringBootTest(classes = DefaultTbelInvokeService.class) public class ScriptCalculatedFieldStateTest { @@ -64,9 +68,13 @@ public class ScriptCalculatedFieldStateTest { @Autowired private TbelInvokeService tbelInvokeService; + @MockBean + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -219,7 +227,7 @@ public class ScriptCalculatedFieldStateTest { ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); argument1.setRefEntityKey(refEntityKey1); argument1.setLimit(5); - argument1.setTimeWindow(30000); + argument1.setTimeWindow(30000L); Argument argument2 = new Argument(); ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d6b384d85b..d803ad2dab 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -32,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -41,7 +45,10 @@ import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) public class SimpleCalculatedFieldStateTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); @@ -55,9 +62,13 @@ public class SimpleCalculatedFieldStateTest { private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; + @Mock + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), null); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index b61c3bc507..9923751db6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.springframework.lang.Nullable; import org.thingsboard.server.common.data.id.EntityId; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Argument { @Nullable @@ -27,7 +29,7 @@ public class Argument { private ReferencedEntityKey refEntityKey; private String defaultValue; - private int limit; - private long timeWindow; + private Integer limit; + private Long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 12cf97338a..b57b19d3ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b4bcc77a17..fd0bf3ceb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -15,18 +15,18 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) public class ReferencedEntityKey { private String key; private ArgumentType type; private AttributeScope scope; - - } From 555acff6b4ced9405b2ef2cfcc37e90a6b4d8fa4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:12:03 +0200 Subject: [PATCH 123/281] fix --- .../panel/calculated-field-argument-panel.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b47d2f073c..fe60162005 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -173,7 +173,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { switch (this.entityId.entityType) { case EntityType.ASSET_PROFILE: return { - deviceTypes: [this.entityName], + assetTypes: [this.entityName], type: AliasFilterType.assetType }; case EntityType.DEVICE_PROFILE: From d185b784274f9ed19302c6bce1e5976fb29bd3ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:57:05 +0200 Subject: [PATCH 124/281] Resolved PR comments --- ...lated-field-arguments-table.component.html | 19 ++++++++------ ...lated-field-arguments-table.component.scss | 9 ++++--- .../calculated-field-dialog.component.ts | 7 +++--- ...lculated-field-argument-panel.component.ts | 25 +++---------------- .../shared/models/calculated-field.models.ts | 21 ++++++++++++++++ 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 24d67d8654..b25809ac82 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -87,19 +87,24 @@ (click)="manageArgument($event, button, $index)" [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> - edit - @if (argumentsFormArray.dirty - && group.get('refEntityKey').get('type').value === ArgumentType.Rolling - && calculatedFieldType() === CalculatedFieldType.SIMPLE) { - - } + + edit +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8f8b1d5d81..3a1f1fbe25 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,9 +19,10 @@ font-size: 14px; } } - .edit-hint { - .mat-mdc-form-field-error { - font-size: 16px; - } +} + +:host { + .field-action { + color: rgba(0, 0, 0, 0.54); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 155a384273..55fc299475 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,7 +36,6 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; -import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -60,10 +59,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => Object.keys(argumentsObj)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index fe60162005..510bdd95f3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -25,7 +25,8 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - CalculatedFieldType + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, filter } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -110,7 +111,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); - this.currentEntityFilter = this.getCurrentEntityFilter(); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -169,26 +170,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.cd.markForCheck(); } - private getCurrentEntityFilter(): EntityFilter { - switch (this.entityId.entityType) { - case EntityType.ASSET_PROFILE: - return { - assetTypes: [this.entityName], - type: AliasFilterType.assetType - }; - case EntityType.DEVICE_PROFILE: - return { - deviceTypes: [this.entityName], - type: AliasFilterType.deviceType - }; - default: - return { - type: AliasFilterType.singleEntity, - singleEntity: this.entityId, - }; - } - } - private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f73926f62b..fac1e9f942 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -20,6 +20,7 @@ import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { debugSettings?: EntityDebugSettings; @@ -133,3 +134,23 @@ export const ArgumentEntityTypeParamsMap =new Map { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} From 33825e5660e5aeb39271904691a503d5c2232fd2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:03 +0200 Subject: [PATCH 125/281] Changed styling --- ...calculated-field-arguments-table.component.html | 7 +++---- ...calculated-field-arguments-table.component.scss | 14 ++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index b25809ac82..4b7e516db0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 3a1f1fbe25..188371b0d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -13,6 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +:host { + .tb-form-table-row-cell-buttons{ + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + :host ::ng-deep { .tb-inline-field { a { @@ -20,9 +28,3 @@ } } } - -:host { - .field-action { - color: rgba(0, 0, 0, 0.54); - } -} From 1ac9d4137773e539880b6ba51063d8c5ddeac499 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:34 +0200 Subject: [PATCH 126/281] formatting --- .../calculated-field-arguments-table.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 188371b0d8..a0c90bba32 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-form-table-row-cell-buttons{ + .tb-form-table-row-cell-buttons { --mat-badge-legacy-small-size-container-size: 8px; --mat-badge-small-size-container-overlap-offset: -5px; --mat-badge-small-size-text-size: 0; From 3818d1cb68d35129e535d96f2ddb098c59812bdb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Feb 2025 12:34:10 +0200 Subject: [PATCH 127/281] debug events(wip) --- .../server/actors/ActorSystemContext.java | 95 ++++++++++++------- .../cf/ctx/state/RocksDBStateService.java | 3 +- .../permission/TenantAdminPermissions.java | 1 + .../src/main/resources/thingsboard.yml | 6 ++ .../dao/cf/BaseCalculatedFieldService.java | 2 +- .../CalculatedFieldDebugEventRepository.java | 2 +- .../main/resources/sql/schema-entities.sql | 4 +- 7 files changed, 72 insertions(+), 41 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 49e2c9c46e..652cdabf77 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -137,6 +137,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j @Component @@ -179,6 +180,8 @@ public class ActorSystemContext { private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap cfDebugPerTenantLimits = new ConcurrentHashMap<>(); + public ConcurrentMap getDebugPerTenantLimits() { return debugPerTenantLimits; } @@ -441,6 +444,11 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + /** * The following Service will be null if we operate in tb-core mode */ @@ -517,11 +525,6 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; - @Lazy - @Autowired(required = false) - @Getter - private ApiLimitService apiLimitService; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -625,6 +628,14 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean cfDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String cfDebugPerTenantLimitsConfiguration; + @Getter @Setter private TbActorSystem actorSystem; @@ -753,37 +764,6 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString(arguments)); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); - } - } - private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> @@ -817,6 +797,49 @@ public class ActorSystemContext { Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + if (cfDebugPerTenantEnabled) { + TbRateLimits rateLimits = cfDebugPerTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(cfDebugPerTenantLimitsConfiguration)); + + if (rateLimits.tryConsume()) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + if (log.isTraceEnabled()) { + log.trace("[{}] Tenant level debug mode rate limit detected: {}", tenantId, calculatedFieldId); + } + } + } + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 371751cc6f..8a6a5c9cb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -61,7 +61,8 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); - if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { rocksDBService.put(toProto(stateId), toProto(stateId, state)); } callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 5b98a56f24..b0e35e57e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,6 +55,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..197f05a5d5 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index d37a4a53c1..b062a2de5e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -62,9 +62,9 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); - updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java index c0bd21ca74..a759babcfb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -35,7 +35,7 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override - @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + @Query("SELECT e FROM CalculatedFieldDebugEventEntity e WHERE " + "e.tenantId = :tenantId " + "AND e.entityId = :entityId " + "AND (:startTime IS NULL OR e.ts >= :startTime) " + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a6d1e8800d..4b395a5768 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -955,10 +955,10 @@ CREATE TABLE IF NOT EXISTS cf_debug_event ( id uuid NOT NULL, tenant_id uuid NOT NULL , ts bigint NOT NULL, - entity_id uuid NOT NULL, + entity_id uuid NOT NULL, -- calculated field id service_id varchar, cf_id uuid NOT NULL, - e_entity_id uuid, + e_entity_id uuid, -- target entity id e_entity_type varchar, e_msg_id uuid, e_msg_type varchar, From 407511be52b1f7de51a74a62cf6d58d7c4ff12aa Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 5 Feb 2025 12:51:50 +0200 Subject: [PATCH 128/281] Cache implementation --- .../server/actors/ActorSystemContext.java | 6 + ...alculatedFieldManagerMessageProcessor.java | 44 ++---- .../service/cf/CalculatedFieldCache.java | 4 - .../cf/DefaultCalculatedFieldCache.java | 60 ++------- ...efaultCalculatedFieldExecutionService.java | 125 ------------------ .../cf/DefaultCalculatedFieldInitService.java | 36 ++--- .../CalculatedFieldEntityProfileCache.java | 35 +++++ ...aultCalculatedFieldEntityProfileCache.java | 86 ++++++++++++ .../cf/cache/TenantEntityProfileCache.java | 118 +++++++++++++++++ .../server/dao/asset/AssetService.java | 3 + .../server/dao/device/DeviceService.java | 3 + .../common/data/ProfileEntityIdInfo.java | 55 ++++++++ .../server/queue/util/AfterStartUp.java | 7 +- .../server/dao/asset/AssetDao.java | 3 + .../server/dao/asset/BaseAssetService.java | 8 ++ .../server/dao/device/DeviceDao.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 8 ++ .../server/dao/sql/asset/JpaAssetDao.java | 12 ++ .../sql/device/AbstractNativeRepository.java | 54 ++++++++ .../device/DefaultNativeAssetRepository.java | 55 ++++++++ .../device/DefaultNativeDeviceRepository.java | 52 ++++---- .../server/dao/sql/device/JpaDeviceDao.java | 7 + .../dao/sql/device/NativeAssetRepository.java | 20 +++ .../sql/device/NativeDeviceRepository.java | 4 +- .../device/NativeProfileEntityRepository.java | 26 ++++ 25 files changed, 578 insertions(+), 257 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..f581d0b266 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,6 +106,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldEntityProfileCache calculatedFieldEntityProfileCache; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index b2ff883236..cd2e5e876f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -26,14 +26,11 @@ import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; @@ -42,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -50,7 +48,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,19 +66,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; protected TbActorCtx ctx; final TenantId tenantId; - private final static int initFetchPackSize = 1024; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); + this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldExecutionService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); @@ -173,8 +170,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); + EntityId profileId = getProfileId(tenantId, entityId); + cfEntityCache.add(tenantId, entityId, profileId); var entityIdFields = getCalculatedFieldsByEntityId(entityId); - var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); @@ -187,6 +186,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); @@ -202,6 +202,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + cfEntityCache.evict(tenantId, msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } @@ -288,7 +289,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = getEntitiesByProfile(entityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { //TODO: no need to do this if we cache all created actors and know which one belong to us; var multiCallback = new MultipleTbCallback(entityIds.size(), callback); @@ -335,7 +336,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var cf = calculatedFields.get(link.cfId()); if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { // iterate over all entities that belong to profile and push the message for corresponding CF - var entityIds = getEntitiesByProfile(targetEntityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); if (!entityIds.isEmpty()) { MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); @@ -392,34 +393,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private Set getEntitiesByProfile(EntityId entityProfileId) { - Set entities = profileEntities.get(entityProfileId); - if (entities == null) { - entities = switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - systemContext.getAssetService().findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - systemContext.getDeviceService().findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); - return entities; - } - private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = getEntitiesByProfile(entityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { var multiCallback = new MultipleTbCallback(entityIds.size(), callback); entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ff3bda5da5..96a3694543 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -31,16 +31,12 @@ public interface CalculatedFieldCache { List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(EntityId entityId); CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); List getCalculatedFieldCtxsByEntityId(EntityId entityId); - Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); - void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..ed5ae23a17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -15,31 +15,28 @@ */ package org.thingsboard.server.service.cf; -import jakarta.annotation.PostConstruct; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -51,32 +48,32 @@ import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + private static final Integer UNKNOWN_PARTITION = -1; + private final Lock calculatedFieldFetchLock = new ReentrantLock(); private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ActorSystemContext actorSystemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; - @PostConstruct + @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); + cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() @@ -84,6 +81,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); + cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); } @Override @@ -96,11 +94,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } - @Override - public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); - } - @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); @@ -139,39 +132,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } - @Override - public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { - Set entities = profileEntities.get(entityProfileId); - if (entities == null) { - calculatedFieldFetchLock.lock(); - try { - entities = profileEntities.get(entityProfileId); - if (entities == null) { - entities = switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> - throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); - return entities; - } - @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..5c4ab5ef6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -277,48 +277,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); -// PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); -// Map> tpiTargetEntityMap = new HashMap<>(); -// -// for (CalculatedField cf : cfs) { -// -// Consumer resolvePartition = entityId -> { -// TopicPartitionInfo tpi; -// try { -// tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); -// if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { -// tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); -// } -// } catch (Exception e) { -// log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", -// entityId, cf.getTenantId(), e.getMessage()); -// } -// }; -// -// EntityId cfEntityId = cf.getEntityId(); -// if (isProfileEntity(cfEntityId)) { -// calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); -// } else { -// resolvePartition.accept(cfEntityId); -// } -// } -// -// for (var entry : tpiTargetEntityMap.entrySet()) { -// for (List partition : Lists.partition(entry.getValue(), 1000)) { -// log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); -// var future = calculatedFieldExecutor.submit(() -> { -// try { -// for (CalculatedFieldEntityCtxId ctxId : partition) { -// restoreState(ctxId.cfId(), ctxId.entityId()); -// } -// } catch (Throwable t) { -// log.error("Unexpected exception while restoring CalculatedField states", t); -// throw t; -// } -// }); -// result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); -// } -// } return result; } @@ -331,89 +289,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } -// @Override -// public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { -// try { -// TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); -// -// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); -// if (tpi.isMyPartition()) { -// log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); -// if (proto.getDeleted()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); -// calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); -// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); -// } -// if (proto.getAdded()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); -// initializeStateForEntityByProfile(entityId, newProfileId, callback); -// } -// if (proto.getUpdated()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); -// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); -// -// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); -// initializeStateForEntityByProfile(entityId, newProfileId, callback); -// } -// } else { -// clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); -// } -// } catch (Exception e) { -// log.trace("Failed to process entity update msg: [{}]", proto, e); -// } -// } - - private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - } - - private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() - .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId())) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); - } - - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { - Map argumentValues = new HashMap<>(commonArguments); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { -// updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); - callback.onSuccess(); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - - @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 87dda7013f..f71617e204 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -17,41 +17,49 @@ package org.thingsboard.server.service.cf; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +@Slf4j @Service @TbRuleEngineComponent @RequiredArgsConstructor public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { - private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldEntityProfileCache entityProfileCache; private final CalculatedFieldStateService stateService; + private final ActorSystemContext actorSystemContext; + private final AssetService assetService; + private final DeviceService deviceService; @Value("${calculated_fields.init_fetch_pack_size:50000}") @Getter private int initFetchPackSize; - @AfterStartUp(order = AfterStartUp.CF_INIT_SERVICE) + @AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) public void initCalculatedFieldDefinitions() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); - //TODO: combine with the DefaultCalculatedFieldCache. - + PageDataIterable deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : deviceIdInfos) { + log.trace("Processing device record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + PageDataIterable assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : assetIdInfos) { + log.trace("Processing asset record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } } @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) @@ -59,6 +67,4 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); } - - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..6b0718eb84 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +import java.util.Collection; + +public interface CalculatedFieldEntityProfileCache extends ApplicationListener { + + void add(TenantId tenantId, EntityId profileId, EntityId entityId); + + void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); + + void evict(TenantId tenantId, EntityId entityId); + + Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..13f95c547d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { + + private static final Integer UNKNOWN = -1; + private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); + private final PartitionService partitionService; + private volatile List myPartitions = Collections.emptyList(); + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + myPartitions = event.getCalculatedFieldsPartitions().stream() + .filter(TopicPartitionInfo::isMyPartition) + .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); + //Naive approach that need to be improved. + tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); + } + + @Override + public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) + .add(profileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + //TODO: make this method atomic; + cache.remove(oldProfileId, entityId); + cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void evict(TenantId tenantId, EntityId entityId) { + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + cache.removeEntityId(entityId); + } + + @Override + public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { + return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java new file mode 100644 index 0000000000..b3499f3cc4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TenantEntityProfileCache { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Map>> allEntities = new HashMap<>(); + private final Map> myEntities = new HashMap<>(); + + public void setMyPartitions(List myPartitions) { + lock.writeLock().lock(); + try { + myEntities.clear(); + myPartitions.forEach(partitionId -> { + var map = allEntities.get(partitionId); + if (map != null) { + map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); + } + }); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeProfileId(EntityId profileId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.remove(profileId)); + // Remove from myEntities + myEntities.remove(profileId); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeEntityId(EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); + // Remove from myEntities + myEntities.values().forEach(set -> set.remove(entityId)); + } finally { + lock.writeLock().unlock(); + } + } + + public void remove(EntityId profileId, EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); + // Remove from myEntities + removeSafely(myEntities, profileId, entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { + lock.writeLock().lock(); + try { + if (mine) { + myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); + } + allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public Collection getMyEntityIdsByProfileId(EntityId profileId) { + lock.readLock().lock(); + try { + var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); + List result = new ArrayList<>(entities.size()); + result.addAll(entities); + return result; + } finally { + lock.readLock().unlock(); + } + } + + private void removeSafely(Map> map, EntityId profileId, EntityId entityId) { + var set = map.get(profileId); + if (set != null) { + set.remove(entityId); + } + } +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 2a9fe08827..15ed81d4c2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -63,6 +64,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 0848e5a1cb..eb018aef1d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -73,6 +74,8 @@ public interface DeviceService extends EntityDaoService { PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java new file mode 100644 index 0000000000..83b56c38a7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@Slf4j +public class ProfileEntityIdInfo implements Serializable, HasTenantId { + + private static final long serialVersionUID = 8532058281983868003L; + + private final TenantId tenantId; + private final EntityId profileId; + private final EntityId entityId; + + private ProfileEntityIdInfo(UUID tenantId, EntityId profileId, EntityId entityId) { + this.tenantId = TenantId.fromUUID(tenantId); + this.profileId = profileId; + this.entityId = entityId; + } + + public static ProfileEntityIdInfo create(UUID tenantId, DeviceProfileId profileId, DeviceId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + + public static ProfileEntityIdInfo create(UUID tenantId, AssetProfileId profileId, AssetId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 70ae853e1e..f97c3b0a1b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,9 +38,10 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; - int CF_INIT_SERVICE = 10; - int CF_STATE_RESTORE_SERVICE = 11; - int CF_CONSUMER_SERVICE = 12; + int CF_READ_PROFILE_ENTITIES_SERVICE = 10; + int CF_READ_CF_SERVICE = 11; + int CF_STATE_RESTORE_SERVICE = 12; + int CF_CONSUMER_SERVICE = 13; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 8e40ec6d87..783a084ef3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -236,4 +237,6 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); PageData> getAllAssetTypes(PageLink pageLink); + + PageData findProfileEntityIdInfos(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 564da127ab..607d09e23c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -26,6 +26,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; @@ -285,6 +286,13 @@ public class BaseAssetService extends AbstractCachedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return assetDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 7c82df7703..b474e0b8e5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -230,5 +231,8 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ffebd5f032..99ae252f79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.audit.ActionType; @@ -386,6 +387,13 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return deviceDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndType, tenantId [{}], type [{}], pageLink [{}]", tenantId, type, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index db6472f825..650d991521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -35,6 +36,8 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeAssetRepository; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; @@ -56,6 +59,9 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Autowired private AssetRepository assetRepository; + @Autowired + private NativeAssetRepository nativeAssetRepository; + @Autowired private AssetProfileRepository assetProfileRepository; @@ -252,6 +258,12 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeAssetRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java new file mode 100644 index 0000000000..bde9278cd4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class AbstractNativeRepository { + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + protected PageData find(String countQuery, String findQuery, Pageable pageable, Function, T> mapper) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(countQuery, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(findQuery, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(mapper).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java new file mode 100644 index 0000000000..43f80e0a86 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.UUID; + +@Repository +@Slf4j +public class DefaultNativeAssetRepository extends AbstractNativeRepository implements NativeAssetRepository { + + private final String COUNT_QUERY = "SELECT count(id) FROM asset;"; + + public DefaultNativeAssetRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + AssetId id = new AssetId((UUID) row.get("id")); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 002c796089..67b45ba70d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -15,50 +15,50 @@ */ package org.thingsboard.server.dao.sql.device; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; -@RequiredArgsConstructor @Repository @Slf4j -public class DefaultNativeDeviceRepository implements NativeDeviceRepository { +public class DefaultNativeDeviceRepository extends AbstractNativeRepository implements NativeDeviceRepository { private final String COUNT_QUERY = "SELECT count(id) FROM device;"; - private final String QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; - private final NamedParameterJdbcTemplate jdbcTemplate; - private final TransactionTemplate transactionTemplate; + + public DefaultNativeDeviceRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } @Override public PageData findDeviceIdInfos(Pageable pageable) { - return transactionTemplate.execute(status -> { - long startTs = System.currentTimeMillis(); - int totalElements = jdbcTemplate.queryForObject(COUNT_QUERY, Collections.emptyMap(), Integer.class); - log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); - startTs = System.currentTimeMillis(); - List> rows = jdbcTemplate.queryForList(String.format(QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); - log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); - int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; - boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); - var data = rows.stream().map(row -> { - UUID id = (UUID) row.get("id"); - var tenantIdObj = row.get("tenantId"); - var customerIdObj = row.get("customerId"); - return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); - }).collect(Collectors.toList()); - return new PageData<>(data, totalPages, totalElements, hasNext); + String DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, DEVICE_ID_INFO_QUERY, pageable, row -> { + UUID id = (UUID) row.get("id"); + var tenantIdObj = row.get("tenantId"); + var customerIdObj = row.get("customerId"); + return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); }); } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + DeviceId id = new DeviceId((UUID) row.get("id")); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index bf0def9fca..74afdca880 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -272,6 +273,12 @@ public class JpaDeviceDao extends JpaAbstractDao implement return nativeDeviceRepository.findDeviceIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeDeviceRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java new file mode 100644 index 0000000000..6a0539ee9d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +public interface NativeAssetRepository extends NativeProfileEntityRepository { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java index f968749461..925bf12618 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java @@ -17,10 +17,12 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.domain.Pageable; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageData; -public interface NativeDeviceRepository { +public interface NativeDeviceRepository extends NativeProfileEntityRepository { PageData findDeviceIdInfos(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java new file mode 100644 index 0000000000..1d3dbbec45 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeProfileEntityRepository { + + PageData findProfileEntityIdInfos(Pageable pageable); + +} From bf9d4d30ae7313a6f150f5a20a6cd484af0555fb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Feb 2025 14:21:41 +0200 Subject: [PATCH 129/281] handled debug error event --- .../CalculatedFieldEntityMessageProcessor.java | 14 ++++++++++---- .../org/thingsboard/common/util/DebugModeUtil.java | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f6bc5b2d63..309dfde2e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -201,10 +201,16 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { if (state.isReady() && ctx.isInitialized()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + try { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } else { callback.onSuccess(); // State was updated but no calculation performed; diff --git a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java index 9ba9038594..d63f846dd2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java @@ -57,4 +57,14 @@ public final class DebugModeUtil { return debugSettings != null && nodeConnections != null && debugSettings.isFailuresEnabled() && nodeConnections.contains(TbNodeConnectionType.FAILURE); } } + + public static boolean isDebugFailuresAvailable(HasDebugSettings debugSettingsAware) { + if (isDebugAllAvailable(debugSettingsAware)) { + return true; + } else { + var debugSettings = debugSettingsAware.getDebugSettings(); + return debugSettings != null && debugSettings.isFailuresEnabled(); + } + } + } From 0332e0f0c953d4e5d7b37d46207177e119631a0d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 5 Feb 2025 16:39:48 +0200 Subject: [PATCH 130/281] Bug fixes for monolith --- .../server/actors/ActorSystemContext.java | 1 + .../CalculatedFieldEntityActorCreator.java | 3 +- ...alculatedFieldManagerMessageProcessor.java | 35 +++++++++++++++---- .../cf/DefaultCalculatedFieldCache.java | 19 ++++++---- ...efaultCalculatedFieldExecutionService.java | 1 - .../cf/cache/TenantEntityProfileCache.java | 4 +++ .../cf/ctx/state/CalculatedFieldCtx.java | 12 ++++--- .../entitiy/EntityStateSourcingListener.java | 8 ++--- .../queue/DefaultTbClusterService.java | 13 ++++--- .../server/actors/TbActorMailbox.java | 3 +- 10 files changed, 70 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 978b2499a4..8f15a6766b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -314,6 +314,7 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; + @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java index c5f8ecf046..1dfe92c4bc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -18,6 +18,7 @@ package org.thingsboard.server.actors.calculatedField; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActor; import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.device.DeviceActor; import org.thingsboard.server.actors.service.ContextBasedCreator; @@ -38,7 +39,7 @@ public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { @Override public TbActorId createActorId() { - return new TbEntityActorId(entityId); + return new TbCalculatedFieldEntityActorId(entityId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 34608337a5..c219544a91 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -50,7 +51,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -175,7 +175,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); EntityId profileId = getProfileId(tenantId, entityId); - cfEntityCache.add(tenantId, entityId, profileId); + cfEntityCache.add(tenantId, profileId, entityId); var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); @@ -233,6 +233,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + addLinks(cf); initCf(cfCtx, callback, false); } } @@ -251,7 +252,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); - List oldCfList = entityIdCalculatedFields.get(newCf.getId()); + List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); boolean found = false; for (CalculatedFieldCtx oldCtx : oldCfList) { @@ -265,10 +266,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!found) { newCfList.add(newCfCtx); } - entityIdCalculatedFields.put(newCf.getId(), newCfList); + entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); + + deleteLinks(oldCfCtx); + addLinks(newCf); + // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { try { newCfCtx.init(); } catch (Exception e) { @@ -276,11 +282,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); } } - initCf(newCfCtx, callback, true); + initCf(newCfCtx, callback, stateChanges); + } else { + callback.onSuccess(); } } } - } private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { @@ -290,6 +297,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); } else { + deleteLinks(cfCtx); + EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { @@ -440,4 +449,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware () -> true); } + private void addLinks(CalculatedField newCf) { + var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); + newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link)); + } + + private void deleteLinks(CalculatedFieldCtx cfCtx) { + var oldCf = cfCtx.getCalculatedField(); + var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); + oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); + } + + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c8f3d98f40..9688f35fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -70,20 +70,25 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { + //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) - ); - cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); + cfs.forEach(cf -> { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + }); + calculatedFields.values().forEach(cf -> { + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); + }); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); + cfls.forEach(link -> { + calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); + actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); + }); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); - cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd8c9b7edd..103997f0c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -136,7 +136,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } }; - private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final CalculatedFieldCache calculatedFieldCache; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java index b3499f3cc4..6aa5c3d4de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.cache; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import java.util.ArrayList; @@ -88,6 +89,9 @@ public class TenantEntityProfileCache { public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { lock.writeLock().lock(); try { + if(EntityType.DEVICE.equals(profileId.getEntityType())){ + throw new RuntimeException("WTF?"); + } if (mine) { myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c483c6ab5d..c3105d6b57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -212,12 +212,16 @@ public class CalculatedFieldCtx { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } - public boolean hasSignificantChanges(CalculatedFieldCtx other) { - boolean entityIdChanged = !entityId.equals(other.entityId); + public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { + boolean expressionChanged = !expression.equals(other.expression); + boolean outputChanged = !output.equals(other.output); + return expressionChanged || outputChanged; + } + + public boolean hasStateChanges(CalculatedFieldCtx other) { boolean typeChanged = !cfType.equals(other.cfType); boolean argumentsChanged = !arguments.equals(other.arguments); - boolean expressionChanged = !expression.equals(other.expression); - return entityIdChanged || typeChanged || argumentsChanged || expressionChanged; + return typeChanged || argumentsChanged; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index d40bf7b130..7265533aab 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -178,11 +178,11 @@ public class EntityStateSourcingListener { } case TENANT_PROFILE -> { TenantProfile tenantProfile = (TenantProfile) event.getEntity(); - tbClusterService.onTenantProfileDelete(tenantProfile, null); + tbClusterService.onTenantProfileDelete(tenantProfile, TbQueueCallback.EMPTY); } case DEVICE -> { Device device = (Device) event.getEntity(); - tbClusterService.onDeviceDeleted(tenantId, device, null); + tbClusterService.onDeviceDeleted(tenantId, device, TbQueueCallback.EMPTY); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -190,11 +190,11 @@ public class EntityStateSourcingListener { } case TB_RESOURCE -> { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); - tbClusterService.onResourceDeleted(tbResource, null); + tbClusterService.onResourceDeleted(tbResource, TbQueueCallback.EMPTY); } case CALCULATED_FIELD -> { CalculatedField calculatedField = (CalculatedField) event.getEntity(); - tbClusterService.onCalculatedFieldDeleted(calculatedField, null); + tbClusterService.onCalculatedFieldDeleted(calculatedField, TbQueueCallback.EMPTY); } default -> { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 944e24480e..fa02435ca0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -727,14 +727,19 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED)); + onCalculatedFieldLifecycleMsg(msg, callback); } @Override public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); - broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + private void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto msg, TbQueueCallback callback) { + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(msg).build(), callback); + broadcastToCore(ToCoreNotificationMsg.newBuilder().setComponentLifecycle(msg).build()); } @Override diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index ee4af20639..a2a26d55e4 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -159,7 +159,8 @@ public final class TbActorMailbox implements TbActorCtx { stopReason = TbActorStopReason.INIT_FAILED; destroy(updateException.getCause()); } catch (Throwable t) { - log.debug("[{}] Failed to process message: {}", selfId, msg, t); + //TODO: revert; + log.error("[{}] Failed to process message: {}", selfId, msg, t); ProcessFailureStrategy strategy = actor.onProcessFailure(msg, t); if (strategy.isStop()) { system.stop(selfId); From 669bfcbb2255dd970705ebe8f5031d2c625d0e2e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 5 Feb 2025 17:57:18 +0200 Subject: [PATCH 131/281] Added calculated field import/export --- .../calculated-fields-table-config.ts | 37 +++++++++++++- .../calculated-fields-table.component.ts | 5 +- .../import-export/import-dialog.component.ts | 2 +- .../import-export/import-export.service.ts | 48 +++++++++++++++++++ .../shared/models/calculated-field.models.ts | 12 +++-- .../assets/locale/locale.constant-en_US.json | 6 +++ 6 files changed, 104 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index dc7aeffb1d..3ff5f3bc7e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -37,6 +37,7 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDialogComponent } from './components/public-api'; +import { ImportExportService } from '@shared/import-export/import-export.service'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -55,7 +56,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.translate.instant('calculated-fields.delete-multiple-title', {count}); this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); + this.addActionDescriptors = [ + { + name: this.translate.instant('calculated-fields.create'), + icon: 'insert_drive_file', + isEnabled: () => true, + onAction: ($event) => this.getTable().addEntity($event) + }, + { + name: this.translate.instant('calculated-fields.import'), + icon: 'file_upload', + isEnabled: () => true, + onAction: () => this.importCalculatedField() + } + ]; this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; @@ -82,6 +98,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, + onAction: (event$, entity) => this.exportCalculatedField(event$, entity), + }, { name: '', nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), @@ -166,6 +188,19 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); + } + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index d212dfcbf7..acd7b18000 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -34,6 +34,7 @@ import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { TbPopoverService } from '@shared/components/popover.service'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { ImportExportService } from '@shared/import-export/import-export.service'; @Component({ selector: 'tb-calculated-fields-table', @@ -59,6 +60,7 @@ export class CalculatedFieldsTableComponent { private popoverService: TbPopoverService, private cd: ChangeDetectorRef, private renderer: Renderer2, + private importExportService: ImportExportService, private destroyRef: DestroyRef) { effect(() => { @@ -73,7 +75,8 @@ export class CalculatedFieldsTableComponent { this.popoverService, this.destroyRef, this.renderer, - this.entityName() + this.entityName(), + this.importExportService ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/shared/import-export/import-dialog.component.ts b/ui-ngx/src/app/shared/import-export/import-dialog.component.ts index e4256229c4..7e46f47978 100644 --- a/ui-ngx/src/app/shared/import-export/import-dialog.component.ts +++ b/ui-ngx/src/app/shared/import-export/import-dialog.component.ts @@ -71,7 +71,7 @@ export class ImportDialogComponent extends DialogComponent, isSingleWidget: boolean, customTitle: string, missingEntityAliases: EntityAliases) => Observable; @@ -116,6 +118,7 @@ export class ImportExportService { private imageService: ImageService, private utils: UtilsService, private itembuffer: ItemBufferService, + private calculatedFieldsService: CalculatedFieldsService, private dialog: MatDialog) { } @@ -171,6 +174,35 @@ export class ImportExportService { ); } + public exportCalculatedField(calculatedFieldId: string): void { + this.calculatedFieldsService.getCalculatedFieldById(calculatedFieldId).subscribe({ + next: (calculatedField) => { + let name = calculatedField.name; + name = name.toLowerCase().replace(/\W/g, '_'); + this.exportToPc(this.prepareCalculatedFieldExport(calculatedField), name); + }, + error: (e) => { + this.handleExportError(e, 'calculated-fields.export-failed-error'); + } + }); + } + + public importCalculatedField(entityId: EntityId): Observable { + return this.openImportDialog('calculated-fields.import', 'calculated-fields.file').pipe( + mergeMap((calculatedField: CalculatedField) => { + if (!this.validateImportedCalculatedField({ entityId, ...calculatedField })) { + this.store.dispatch(new ActionNotificationShow( + {message: this.translate.instant('calculated-fields.invalid-file-error'), + type: 'error'})); + throw new Error('Invalid calculated field file'); + } else { + return this.calculatedFieldsService.saveCalculatedField(this.prepareImport({ entityId, ...calculatedField })); + } + }), + catchError(() => of(null)), + ); + } + public exportDashboard(dashboardId: string) { this.getIncludeResourcesPreference('includeResourcesInExportDashboard').subscribe(includeResources => { this.openExportDialog('dashboard.export', 'dashboard.export-prompt', includeResources).subscribe(result => { @@ -957,6 +989,17 @@ export class ImportExportService { } } + private validateImportedCalculatedField(calculatedField: CalculatedField): boolean { + const { name, configuration, entityId } = calculatedField; + return isNotEmptyStr(name) + && isDefined(configuration) + && isDefined(entityId?.id) + && !!Object.keys(configuration.arguments).length + && isDefined(configuration.expression) + && isDefined(configuration.output) + && isNotEmptyStr(configuration.output.name); + } + private validateImportedImage(image: ImageExportData): boolean { return !(!isNotEmptyStr(image.data) || !isNotEmptyStr(image.title) @@ -1209,6 +1252,11 @@ export class ImportExportService { return profile; } + private prepareCalculatedFieldExport(calculatedField: CalculatedField): CalculatedField { + delete calculatedField.entityId; + return this.prepareExport(calculatedField); + } + private prepareExport(data: any): any { const exportedData = deepClone(data); if (isDefined(exportedData.id)) { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index fac1e9f942..3a86b12018 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -15,16 +15,15 @@ /// import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType } from '@shared/models/alias.models'; -export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId, ExportableEntity { debugSettings?: EntityDebugSettings; - externalId?: string; configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; entityId: EntityId; @@ -46,6 +45,13 @@ export interface CalculatedFieldConfiguration { type: CalculatedFieldType; expression: string; arguments: Record; + output: CalculatedFieldOutput; +} + +export interface CalculatedFieldOutput { + type: OutputType; + name: string; + scope?: AttributeScope; } export enum ArgumentEntityType { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cb6f124b1d..469bc9a00f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1043,6 +1043,12 @@ "asset-name": "Asset name", "timeseries": "Time series", "output": "Output", + "create": "Create new calculated field", + "file": "Calculated field file", + "invalid-file-error": "Invalid file format. Please make sure the file is a valid JSON file.", + "import": "Import calculated field", + "export": "Export calculated field", + "export-failed-error": "Unable to export calculated field: {{error}}", "output-type": "Output type", "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", From 2214bf8ef23a6cb824926d174d92303d2bc9da54 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:47:52 +0200 Subject: [PATCH 132/281] Calculated Fields Debug Events dialog basic implementation --- .../server/common/data/event/EventFilter.java | 3 +- .../calculated-fields-table-config.ts | 32 ++++++- ...lculated-field-debug-dialog.component.html | 47 ++++++++++ ...calculated-field-debug-dialog.component.ts | 53 +++++++++++ .../calculated-field-dialog.component.html | 1 + .../calculated-field-dialog.component.ts | 8 ++ .../components/public-api.ts | 1 + .../entity-debug-settings-button.component.ts | 6 +- ...entity-debug-settings-panel.component.html | 40 ++++++--- .../entity-debug-settings-panel.component.ts | 5 +- .../components/event/event-table-config.ts | 89 +++++++++++++++++++ .../home/components/home-components.module.ts | 5 ++ .../shared/models/calculated-field.models.ts | 14 ++- ui-ngx/src/app/shared/models/entity.models.ts | 9 ++ ui-ngx/src/app/shared/models/event.models.ts | 16 +++- .../assets/locale/locale.constant-en_US.json | 5 ++ 16 files changed, 310 insertions(+), 24 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 6d2a110cf7..4c9791e3fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 3ff5f3bc7e..c5e455583d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -35,8 +35,12 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; -import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; -import { CalculatedFieldDialogComponent } from './components/public-api'; +import { + CalculatedField, + CalculatedFieldDebugDialogData, + CalculatedFieldDialogData +} from '@shared/models/calculated-field.models'; +import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -46,6 +50,14 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog({...this.additionalDebugActionConfig.data, id }), + }; const { viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); @@ -140,6 +156,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }) + .afterClosed() + .subscribe(); + } + private exportCalculatedField($event: Event, calculatedField: CalculatedField): void { if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html new file mode 100644 index 0000000000..1b61a9da4a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -0,0 +1,47 @@ + +
+ +

{{ 'calculated-fields.debugging' | translate}}

+ + + +
+ +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts new file mode 100644 index 0000000000..8efb21dbf2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -0,0 +1,53 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, Inject, ViewChild } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { DebugEventType, EventType } from '@shared/models/event.models'; +import { EventTableComponent } from '@home/components/event/event-table.component'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-debug-dialog', + templateUrl: './calculated-field-debug-dialog.component.html', +}) +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; + + readonly DebugEventType = DebugEventType; + readonly debugEventTypes = DebugEventType; + readonly EventType = EventType; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, + protected dialogRef: MatDialogRef) { + super(store, router, dialogRef); + } + + ngAfterViewInit(): void { + this.eventsTable.entitiesTable.updateData(); + } + + cancel(): void { + this.dialogRef.close(null); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 37e96cfd91..ac60ae8178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -51,6 +51,7 @@ [class.mb-5]="fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched" [entityLabel]="'debug-settings.calculated-field' | translate" [debugLimitsConfiguration]="data.debugLimitsConfiguration" + [additionalActionConfig]="additionalDebugActionConfig" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 55fc299475..3575223764 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -65,6 +65,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(argumentsObj)) ); + additionalDebugActionConfig = this.data.value?.id ? { + ...this.data.additionalDebugActionConfig, + action: () => this.data.additionalDebugActionConfig.action({ + ...this.data.additionalDebugActionConfig.data, + id: this.data.value.id, + }), + } : null; + readonly OutputTypeTranslations = OutputTypeTranslations; readonly OutputType = OutputType; readonly AttributeScope = AttributeScope; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index bc89e4dc6f..78b8862c2e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -17,3 +17,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; +export * from './debug-dialog/calculated-field-debug-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts index 597fd79b8e..7e06568489 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts @@ -32,7 +32,7 @@ import { EntityDebugSettingsPanelComponent } from './entity-debug-settings-panel import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { BehaviorSubject, of, shareReplay, timer } from 'rxjs'; import { SECOND, MINUTE } from '@shared/models/time/time.models'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { map, switchMap, takeWhile } from 'rxjs/operators'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { AppState } from '@core/core.state'; @@ -61,6 +61,7 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor @Input() debugLimitsConfiguration: string; @Input() entityLabel: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; debugSettingsFormGroup = this.fb.group({ failuresEnabled: [false], @@ -133,7 +134,8 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor ...debugSettings, maxDebugModeDuration: this.maxDebugModeDuration, debugLimitsConfiguration: this.debugLimitsConfiguration, - entityLabel: this.entityLabel + entityLabel: this.entityLabel, + additionalActionConfig: this.additionalActionConfig, }, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index 7576d4b2fe..dcc3355890 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -48,20 +48,32 @@ -
- - +
+
+ @if (additionalActionConfig) { + + } +
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts index 72748e27e2..9e8ac1ded6 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts @@ -21,7 +21,7 @@ import { Component, EventEmitter, Input, - OnInit + OnInit, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -32,7 +32,7 @@ import { SECOND } from '@shared/models/time/time.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { of, shareReplay, timer } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs/operators'; @Component({ @@ -54,6 +54,7 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements @Input() allEnabledUntil = 0; @Input() maxDebugModeDuration: number; @Input() debugLimitsConfiguration: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; onFailuresControl = this.fb.control(false); debugAllControl = this.fb.control(false); diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 953d1d17c1..8de4a9f256 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -355,6 +355,86 @@ export class EventTableConfig extends EntityTableConfig { '48px') ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.columns[0].width = '160px'; + this.columns.push( + new EntityTableColumn('entityId', 'event.entity-id', '150px', + (entity) => entity.body.entityId, + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-entity-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.entityId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageId', 'event.message-id', '150px', + (entity) => entity.body.msgId ?? '', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-message-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.msgId ?? '', + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageType', 'event.message-type', '150px', + (entity) => entity.body.msgType ?? '', + () => ({padding: '0 12px 0 0'}), + false + ), + new EntityActionTableColumn('arguments', 'event.arguments', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.arguments !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.arguments, + 'event.arguments', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('result', 'event.result', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.result !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.result, + 'event.result', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '100px' + ) + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -446,6 +526,15 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.filterColumns.push( + {key: 'entityId', title: 'event.entity-id'}, + {key: 'messageId', title: 'event.message-id'}, + {key: 'messageType', title: 'event.message-type'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f60ea15407..b7e35c5406 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -195,6 +195,9 @@ import { import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; @NgModule({ declarations: @@ -343,6 +346,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], imports: [ CommonModule, @@ -485,6 +489,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 3a86b12018..31dca236a4 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -14,7 +14,12 @@ /// limitations under the License. /// -import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; +import { + AdditionalDebugActionConfig, + EntityDebugSettings, + HasTenantId, + HasVersion +} from '@shared/models/entity.models'; import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; @@ -128,6 +133,13 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; + additionalDebugActionConfig: AdditionalDebugActionConfig; +} + +export interface CalculatedFieldDebugDialogData { + id?: CalculatedFieldId; + entityId: EntityId; + tenantId: string; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 9934da65aa..db00955340 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,6 +21,7 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -203,4 +204,12 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; + +export interface AdditionalDebugActionConfig { + action?: (data?: AdditionalDebugActionConfigData) => void; + title: string; + data: AdditionalDebugActionConfigData; +} + export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index a1abb8d15c..8a5009f758 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -29,7 +29,8 @@ export enum EventType { export enum DebugEventType { DEBUG_RULE_NODE = 'DEBUG_RULE_NODE', - DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN' + DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN', + DEBUG_CALCULATED_FIELD = 'DEBUG_CALCULATED_FIELD' } export const eventTypeTranslations = new Map( @@ -39,6 +40,7 @@ export const eventTypeTranslations = new Map [EventType.STATS, 'event.type-stats'], [DebugEventType.DEBUG_RULE_NODE, 'event.type-debug-rule-node'], [DebugEventType.DEBUG_RULE_CHAIN, 'event.type-debug-rule-chain'], + [DebugEventType.DEBUG_CALCULATED_FIELD, 'event.type-debug-calculated-field'], ] ); @@ -80,7 +82,7 @@ export interface DebugRuleChainEventBody extends BaseEventBody { error?: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody & CalculatedFieldEventBody; export interface Event extends BaseData { tenantId: TenantId; @@ -90,6 +92,16 @@ export interface Event extends BaseData { body: EventBody; } +export interface CalculatedFieldEventBody extends BaseFilterEventBody { + calculatedFieldId: string; + entityId: string; + entityType: EntityType; + arguments: string, + result: string, + msgId: string; + msgType: string; +} + export interface BaseFilterEventBody { server?: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 469bc9a00f..9bdf533fef 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1014,6 +1014,7 @@ "script": "Script" }, "arguments": "Arguments", + "debugging": "Calculated field debugging", "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", @@ -1026,6 +1027,7 @@ "argument-customer": "Customer", "argument-tenant": "Current tenant", "argument-type": "Argument type", + "see-debug-events": "See debug events", "attribute": "Attribute", "timeseries-key": "Time series key", "device-name": "Device name", @@ -2710,6 +2712,9 @@ "type-stats": "Statistics", "type-debug-rule-node": "Debug", "type-debug-rule-chain": "Debug", + "type-debug-calculated-field": "Debug", + "arguments": "Arguments", + "result": "Result", "no-events-prompt": "No events found", "error": "Error", "alarm": "Alarm", From 7ad8913b16b6d2ac0b99e57c0a7fa1ea520b4f79 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:52:29 +0200 Subject: [PATCH 133/281] refactoring --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 1b61a9da4a..715cf0572d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data?.id" + [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" /> From c8b7b1c3a243ef021080014cc54f7a83d5ca44db Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:53:55 +0200 Subject: [PATCH 134/281] refactoring --- .../app/modules/home/components/event/event-table-config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 8de4a9f256..da391d3ecb 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '150px', - (entity) => entity.body.msgId ?? '', + (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -394,12 +394,12 @@ export class EventTableConfig extends EntityTableConfig { color: 'rgba(0,0,0,.87)' }, isEnabled: () => true, - onAction: ($event, entity) => entity.body.msgId ?? '', + onAction: ($event, entity) => entity.body.msgId, type: CellActionDescriptorType.COPY_BUTTON } ), new EntityTableColumn('messageType', 'event.message-type', '150px', - (entity) => entity.body.msgType ?? '', + (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false ), From 5354eaa80b7ffceece43aa7ea3d8d0e52e875aa1 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 16:10:26 +0200 Subject: [PATCH 135/281] Adjusted sizing --- .../calculated-field-debug-dialog.component.html | 2 +- .../home/components/event/event-table-config.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 715cf0572d..25f2a11f16 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index da391d3ecb..ae98560969 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -358,7 +358,7 @@ export class EventTableConfig extends EntityTableConfig { case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '160px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '150px', + new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => entity.body.entityId, () => ({padding: '0 12px 0 0'}), false, @@ -378,7 +378,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '150px', + new EntityTableColumn('messageId', 'event.message-id', '85px', (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, @@ -398,7 +398,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageType', 'event.message-type', '150px', + new EntityTableColumn('messageType', 'event.message-type', '100px', (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false @@ -411,7 +411,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.arguments, 'event.arguments', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('result', 'event.result', { @@ -421,7 +421,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.result, 'event.result', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('error', 'event.error', { @@ -431,7 +431,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.error, 'event.error') }, - '100px' + '48px' ) ); break; From 7fcd948071ffbefb0ff654758e58346a05150546 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 16:11:58 +0200 Subject: [PATCH 136/281] fixed error when no telemetry in db --- .../cf/DefaultCalculatedFieldCache.java | 3 ++- ...efaultCalculatedFieldExecutionService.java | 5 ++++- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/RocksDBStateService.java | 19 +++++++++++++++---- .../ctx/state/SingleValueArgumentEntry.java | 9 ++++++++- .../state/SingleValueArgumentEntryTest.java | 4 ++++ .../server/common/data/event/EventFilter.java | 3 ++- 7 files changed, 36 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 9688f35fef..8fffa0029c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -117,6 +117,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); + ctx.init(); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 103997f0c7..43e85d87e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; @@ -68,7 +69,6 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -447,6 +447,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); + if (StringUtils.isBlank(defaultValue)) { + return new StringDataEntry(key, null); + } if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 86d83a1f70..f1ce8038c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -52,7 +52,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null) { + if (existingEntry == null || existingEntry == SingleValueArgumentEntry.EMPTY || existingEntry == TsRollingArgumentEntry.EMPTY) { validateNewEntry(newEntry); arguments.put(key, newEntry); stateUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 8a6a5c9cb7..b2e33e1705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -63,7 +63,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); } @@ -111,8 +111,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName) - .setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + .setArgName(argName); + + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); @@ -122,7 +125,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } return builder.build(); } @@ -151,6 +156,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } TsValueProto tsValueProto = proto.getValue(); long ts = tsValueProto.getTs(); BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); @@ -158,6 +166,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } TreeMap tsRecords = new TreeMap<>(); proto.getTsValueList().forEach(tsValueProto -> { BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 0832e53e5e..d7e5ddd017 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -94,10 +94,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; + + // TODO: should we persist updated ts and version values? + BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); + if (this.kvEntryValue.getValue().equals(newValue.getValue())) { + return false; + } + this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } + } else { throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 203d7b3d71..13651e852d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -69,4 +69,8 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 6d2a110cf7..4c9791e3fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { From ff647aedc7c95c205fd4d27b52867b140504f2d4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:01:33 +0200 Subject: [PATCH 137/281] Improved context passing --- .../calculated-fields-table-config.ts | 17 +++++++++-------- .../dialog/calculated-field-dialog.component.ts | 5 +---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +----- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c5e455583d..026c249159 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -42,6 +42,7 @@ import { } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -52,11 +53,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -140,7 +137,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog({...this.additionalDebugActionConfig.data, id }), + action: () => this.openDebugDialog(id) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -206,11 +203,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data + data: { + tenantId: this.tenantId, + entityId: this.entityId, + id + } }) .afterClosed() .subscribe(); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 3575223764..d5bf243430 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -67,10 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action({ - ...this.data.additionalDebugActionConfig.data, - id: this.data.value.id, - }), + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index db00955340..b9d2402850 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,7 +21,6 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -204,12 +203,9 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; - export interface AdditionalDebugActionConfig { - action?: (data?: AdditionalDebugActionConfigData) => void; + action?: (id?: EntityId) => void; title: string; - data: AdditionalDebugActionConfigData; } export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; From ebab88ac6a06d04d3141c53107b927cd1257bcef Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 17:06:21 +0200 Subject: [PATCH 138/281] added logs --- ...CalculatedFieldEntityMessageProcessor.java | 7 +++++ ...alculatedFieldManagerMessageProcessor.java | 28 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 309dfde2e9..544dea44c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -83,12 +83,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); } public void process(EntityInitCalculatedFieldMsg msg) { + log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); var cfCtx = msg.getCtx(); if (msg.isForceReinit()) { + log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId()); states.remove(cfCtx.getCfId()); } var cfState = getOrInitState(cfCtx); @@ -96,6 +99,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldEntityDeleteMsg msg) { + log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); @@ -110,6 +114,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldTelemetryMsg msg) { + log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); @@ -124,6 +129,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var ctx = msg.getCtx(); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); @@ -169,6 +175,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { + log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index c219544a91..7b03ed2938 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -91,11 +91,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + log.info("[{}] Processing CF init message.", msg.getCf().getId()); var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -108,6 +110,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -121,6 +124,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedField != null) { msg.getState().setRequiredArguments(calculatedField.getArgNames()); + log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); @@ -128,6 +132,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); var event = msg.getData().getEvent(); switch (entityType) { @@ -207,6 +212,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { cfEntityCache.evict(tenantId, msg.getEntityId()); + log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } @@ -225,6 +231,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -251,6 +258,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -318,12 +333,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); + log.info("Received telemetry msg from entity [{}]", entityId); // 2 = 1 for CF processing + 1 for links processing MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + log.info("Pushing telemetry msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); } else { callback.onSuccess(); @@ -340,6 +357,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); + log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); var proto = msg.getProto(); var linksList = proto.getLinksList(); for (var linkProto : linksList) { @@ -353,12 +371,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!entityIds.isEmpty()) { MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); - entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + entityIds.forEach(entityId -> { + log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + }); } else { msg.getCallback().onSuccess(); } } else { - // push the message to specific entity; + log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); getOrCreateActor(targetEntityId).tell(newMsg); } @@ -423,10 +444,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + log.info("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); } private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + log.info("Pushing entity init CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); } @@ -460,5 +483,4 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } - } From dfb22314e4783af3cdacc0d8f490ffe6741e0474 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:04 +0200 Subject: [PATCH 139/281] Improved styling --- ...lculated-field-debug-dialog.component.html | 4 ++-- ...lculated-field-debug-dialog.component.scss | 24 +++++++++++++++++++ ...calculated-field-debug-dialog.component.ts | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 25f2a11f16..6166765bc2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
implements AfterViewInit { From 0c226afbc8177968836974ecfb0572cffd59ea78 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:45 +0200 Subject: [PATCH 140/281] class order --- .../debug-dialog/calculated-field-debug-dialog.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 6166765bc2..3d92ba5454 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
Date: Thu, 6 Feb 2025 17:14:16 +0200 Subject: [PATCH 141/281] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- .../debug-dialog/calculated-field-debug-dialog.component.scss | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 3d92ba5454..91da675fea 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 19bf072b11..1e33c9371e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,6 +16,7 @@ :host { .debug-dialog-container { height: 77vh; + min-width: 90vw; .debug-dialog-content { border-radius: 0; From c167c6969fa82b8e83a17c6fe7627e25c9786e25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:17:10 +0200 Subject: [PATCH 142/281] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 1e33c9371e..23aa4070d7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,7 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 90vw; + min-width: 80vw; .debug-dialog-content { border-radius: 0; From bee6375572e42f6f39e6db4fb45e0647a38efd19 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:31:59 +0200 Subject: [PATCH 143/281] Improved styling --- .../modules/home/components/event/event-table-config.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index ae98560969..a937e42bb8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -356,10 +356,11 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_CALCULATED_FIELD: - this.columns[0].width = '160px'; + this.columns[0].width = '80px'; + this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => entity.body.entityId, + (entity) => `${entity.body.entityId.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -379,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => entity.body.msgId, + (entity) => `${entity.body.msgId?.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), From c582740175ce95814cc80e458505f2b181c29acc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 09:56:42 +0200 Subject: [PATCH 144/281] added endpoint to test script --- .../controller/CalculatedFieldController.java | 84 +++++++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 3 + .../CalculatedFieldTbelScriptEngine.java | 7 ++ 3 files changed, 94 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 96a97aeeff..435e46069d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; @@ -29,6 +32,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -38,16 +43,27 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -59,9 +75,29 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI public class CalculatedFieldController extends BaseController { private final TbCalculatedFieldService tbCalculatedFieldService; + private final TbelInvokeService tbelInvokeService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + public static final int TIMEOUT = 20; + + private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + + " \"argNames\": [\"temperature\", \"humidity\"],\n" + + " \"arguments\": {\n" + + " \"temperature\": {\n" + + " \"14327856345\": 22.4,\n" + + " \"14327857298\": 21.9,\n" + + " \"14327857510\": 22.0\n" + + " },\n" + + " \"humidity\": 42\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + "\n\n Expected result JSON contains \"output\" and \"error\"."; + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + "The newly created Calculated Field Id will be present in the response. " + @@ -128,4 +164,52 @@ public class CalculatedFieldController extends BaseController { tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } + @ApiOperation(value = "Test Script expression", + notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) + @ResponseBody + public JsonNode testScript( + @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") + @RequestBody JsonNode inputParams) { + String expression = inputParams.get("expression").asText(); + String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + }), + Collections.emptyMap() + ); + + String output = ""; + String errorText = ""; + + try { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + getTenantId(), + tbelInvokeService, + expression, + argNames + ); + + Object[] args = Arrays.stream(argNames) + .map(arguments::get) + .toArray(); + + JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); + output = JacksonUtil.toString(json); + } catch (Exception e) { + log.error("Error evaluating expression", e); + errorText = e.getMessage(); + } + + ObjectNode result = JacksonUtil.newObjectNode(); + result.put("output", output); + result.put("error", errorText); + return result; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 779f52c5d6..6bea6ce705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import java.util.Map; @@ -25,6 +26,8 @@ public interface CalculatedFieldScriptEngine { ListenableFuture> executeToMapAsync(Object[] args); + ListenableFuture executeJsonAsync(Object[] args); + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7ac032573f..9e05e05970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -74,6 +76,11 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture executeJsonAsync(Object[] args) { + return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> executeToMapTransform(Object result) { if (result instanceof Map) { From c27c82b80d5943b169b6bbc37715cb1d0fe8790e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:01:02 +0200 Subject: [PATCH 145/281] updated test expression endpoint --- .../server/controller/CalculatedFieldController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 435e46069d..5c8b40dcad 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -48,7 +48,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngi import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -85,7 +85,6 @@ public class CalculatedFieldController extends BaseController { + MARKDOWN_CODE_BLOCK_START + "{\n" + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + - " \"argNames\": [\"temperature\", \"humidity\"],\n" + " \"arguments\": {\n" + " \"temperature\": {\n" + " \"14327856345\": 22.4,\n" + @@ -173,12 +172,12 @@ public class CalculatedFieldController extends BaseController { @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") @RequestBody JsonNode inputParams) { String expression = inputParams.get("expression").asText(); - String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); Map arguments = Objects.requireNonNullElse( JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { }), Collections.emptyMap() ); + ArrayList argNames = new ArrayList<>(arguments.keySet()); String output = ""; String errorText = ""; @@ -192,10 +191,10 @@ public class CalculatedFieldController extends BaseController { getTenantId(), tbelInvokeService, expression, - argNames + argNames.toArray(String[]::new) ); - Object[] args = Arrays.stream(argNames) + Object[] args = argNames.stream() .map(arguments::get) .toArray(); From fe4d2ba49759f8dfcf9313e4777bce69235c7c7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:24:39 +0200 Subject: [PATCH 146/281] added tests --- ...alculatedFieldManagerMessageProcessor.java | 10 +- .../cf/CalculatedFieldIntegrationTest.java | 266 ++++++++++++++++++ .../CalculatedFieldConfiguration.java | 2 + 3 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7b03ed2938..6a6edae768 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -258,14 +258,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); - try { - newCfCtx.init(); - } catch (Exception e) { - log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -289,7 +281,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); - if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { try { newCfCtx.init(); } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java new file mode 100644 index 0000000000..fda0724b1b --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.controller.CalculatedFieldControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + + @BeforeEach + void setUp() throws Exception { + loginTenantAdmin(); + } + + @Test + public void testSimpleCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); + + JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + + // update telemetry -> recalculate state + JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + + Thread.sleep(300); + + ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTempAfterUpdate).isNotNull(); + assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF output -> perform calculation with updated output + Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); + savedOutput.setType(OutputType.ATTRIBUTES); + savedOutput.setScope(AttributeScope.SERVER_SCOPE); + savedOutput.setName("temperatureF"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF argument -> perform calculation with new argument + + Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); + savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateArg).isNotNull(); + assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + + // update CF expression -> perform calculation with new expression + savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateExpression).isNotNull(); + assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = new Asset(); + asset1.setName("Test asset 1"); + asset1.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + + JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); + + Asset asset2 = new Asset(); + asset2.setName("Test asset 2"); + asset2.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); + + JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(assetProfile.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("z = x + y"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(testDevice.getId()); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument2.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("x", argument2, "y", argument1)); + + config.setExpression("x + y"); + + Output output = new Output(); + output.setName("z"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF and perform initial calculation + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + + // result of asset 2 + ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + + // update device telemetry -> recalculate state for all assets + JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ1).isNotNull(); + assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + + // result of asset 2 + ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ2).isNotNull(); + assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); + +// // update asset 1 telemetry -> recalculate state only for asset 1 +// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 (no changes) +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); +// +// // update asset 2 telemetry -> recalculate state only for asset 2 +// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 (no changes) +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 8f56bf491d..9bf3d728aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -46,6 +46,8 @@ public interface CalculatedFieldConfiguration { String getExpression(); + void setExpression(String expression); + Output getOutput(); @JsonIgnore From 96e292fac5cb76511b645553ed83fe7b6b4b53d7 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 15:31:21 +0200 Subject: [PATCH 147/281] Calculated field fixes and improvements --- ...lated-field-arguments-table.component.html | 7 ++-- ...culated-field-arguments-table.component.ts | 20 ++++------ .../calculated-field-dialog.component.html | 2 +- ...ulated-field-argument-panel.component.html | 37 +++++++++++-------- ...ulated-field-argument-panel.component.scss | 22 +++++++++++ ...lculated-field-argument-panel.component.ts | 24 +++++++++++- .../import-export/import-export.service.ts | 1 - .../assets/locale/locale.constant-en_US.json | 1 + 8 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4b7e516db0..d8a6c7cdb0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -88,9 +88,8 @@ [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> {{ 'calculated-fields.no-arguments' | translate }} }
- @if (errorText && this.argumentsFormArray.dirty) { + @if (errorText) { }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index a004e3db6f..171c382b7a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,9 +17,7 @@ import { ChangeDetectorRef, Component, - effect, forwardRef, - input, Input, OnChanges, Renderer2, @@ -77,8 +75,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; - - calculatedFieldType = input() + @Input() calculatedFieldType: CalculatedFieldType; errorText = ''; argumentsFormArray = this.fb.array([]); @@ -103,17 +100,12 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => { - if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { - this.argumentsFormArray.updateValueAndValidity(); - } - }); } ngOnChanges(changes: SimpleChanges): void { if (changes.calculatedFieldType?.previousValue && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { - this.argumentsFormArray.markAsDirty(); + this.argumentsFormArray.updateValueAndValidity(); } } @@ -142,14 +134,16 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { + const argumentObj = this.argumentsFormArray.at(index)?.getRawValue() ?? {}; const ctx = { index, - argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + argument: argumentObj, entityId: this.entityId, - calculatedFieldType: this.calculatedFieldType(), + calculatedFieldType: this.calculatedFieldType, buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, + argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -171,7 +165,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private updateErrorText(): void { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; } else if (!this.argumentsFormArray.controls.length) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ac60ae8178..1d708f060a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
-
{{ 'calculated-fields.arguments' | translate }}
+
{{ 'calculated-fields.arguments' | translate }}*
warning - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) {
} @else { -
-
{{ 'calculated-fields.attribute-scope' | translate }}
- - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if (entityType === ArgumentEntityType.Device - || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { + @if (isDeviceEntity) { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} + {{ 'calculated-fields.client-attributes' | translate }} {{ 'calculated-fields.shared-attributes' | translate }} - } - - -
+ + +
+ }
{{ 'calculated-fields.attribute-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..45c17628d5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .time-window-field { + .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { + border-left: 1px solid rgba(0, 0, 0, 0) !important; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 510bdd95f3..1632bd9311 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -52,11 +53,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; + @Input() argumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, this.uniqNameRequired(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -109,6 +111,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } + get isDeviceEntity(): boolean { + return this.entityType === ArgumentEntityType.Device + || (this.entityType === ArgumentEntityType.Current + && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) + } + ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); @@ -188,9 +196,21 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + if (!this.isDeviceEntity) { + this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); + } }); } + private uniqNameRequired(): ValidatorFn { + return (control: UntypedFormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + private observeEntityKeyChanges(): void { this.argumentFormGroup.get('refEntityKey').get('type').valueChanges .pipe(takeUntilDestroyed()) diff --git a/ui-ngx/src/app/shared/import-export/import-export.service.ts b/ui-ngx/src/app/shared/import-export/import-export.service.ts index 0a49acf8b6..18000bcd4f 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.service.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.service.ts @@ -997,7 +997,6 @@ export class ImportExportService { && !!Object.keys(configuration.arguments).length && isDefined(configuration.expression) && isDefined(configuration.output) - && isNotEmptyStr(configuration.output.name); } private validateImportedImage(image: ImageExportData): boolean { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9bdf533fef..d17409e52d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1064,6 +1064,7 @@ "expression-max-length": "Expression length should be less than 255 characters.", "argument-name-required": "Argument name is required.", "argument-name-pattern": "Argument name is invalid.", + "argument-name-duplicate": "Argument with such name already exists.", "argument-name-max-length": "Argument name should be less than 256 characters.", "argument-type-required": "Argument type is required." } From 8e3c4dc18c3de285f64b111fe62affb03710416a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 7 Feb 2025 15:46:25 +0200 Subject: [PATCH 148/281] Cluster mode fixes --- .../server/actors/ActorSystemContext.java | 6 ++ ...CalculatedFieldEntityMessageProcessor.java | 26 ++++--- ...alculatedFieldManagerMessageProcessor.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 3 - ...efaultCalculatedFieldExecutionService.java | 11 --- .../cf/DefaultCalculatedFieldInitService.java | 9 --- .../server/service/cf/RocksDBService.java | 3 +- ...aultCalculatedFieldEntityProfileCache.java | 1 + .../cf/ctx/CalculatedFieldStateService.java | 4 -- .../KafkaCalculatedFieldStateService.java | 68 +++++++++++++++++++ ...> RocksDBCalculatedFieldStateService.java} | 17 +++-- .../queue/DefaultTbClusterService.java | 10 +-- 12 files changed, 116 insertions(+), 47 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{RocksDBStateService.java => RocksDBCalculatedFieldStateService.java} (92%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 8f15a6766b..c714ed7dab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -108,6 +108,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -527,6 +528,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldStateService calculatedFieldStateService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 544dea44c9..0d327adca9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,6 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,6 +68,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; final CalculatedFieldExecutionService cfService; + final CalculatedFieldStateService cfStateService; TbActorCtx ctx; Map states = new HashMap<>(); @@ -76,6 +78,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.tenantId = tenantId; this.entityId = entityId; this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); } void init(TbActorCtx ctx) { @@ -102,13 +105,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); - states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); ctx.stop(ctx.getSelf()); } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var state = states.remove(cfId); if (state != null) { - cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } } } @@ -178,8 +181,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } - CalculatedFieldState state = getOrInitState(ctx); - if (state.updateState(newArgValues)) { + CalculatedFieldState state = states.get(ctx.getCfId()); + boolean justRestored = false; + if (state == null) { + state = getOrInitState(ctx); + justRestored = true; + } + if (state.updateState(newArgValues) || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); @@ -222,7 +230,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfStateService.persistState(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 6a6edae768..cdc31ed93c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -42,6 +42,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -68,6 +69,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; @@ -80,6 +82,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); @@ -127,7 +130,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); + cfStateService.removeState(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 393fbd3ec2..47fd560b4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,13 +43,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); - void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 43e85d87e9..e84e785a94 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -141,7 +141,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; @@ -263,16 +262,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(ctx, stateId, state, callback); - } - - @Override - public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { - stateService.removeState(calculatedFieldEntityCtxId, callback); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index f71617e204..2afd6d8238 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @Slf4j @Service @@ -38,9 +37,6 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { private final CalculatedFieldEntityProfileCache entityProfileCache; - private final CalculatedFieldStateService stateService; - - private final ActorSystemContext actorSystemContext; private final AssetService assetService; private final DeviceService deviceService; @@ -62,9 +58,4 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer } } - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index fe800f61ad..7181cc43ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -32,7 +33,7 @@ import java.util.Map; @Service @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='monolith'") +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 13f95c547d..866cc86f24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -39,6 +39,7 @@ import java.util.stream.Collectors; @Service @Slf4j @RequiredArgsConstructor +//TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { private static final Integer UNKNOWN = -1; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index e822d52767..ce1562c735 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -19,12 +19,8 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import java.util.Map; - public interface CalculatedFieldStateService { - Map restoreStates(); - void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java new file mode 100644 index 0000000000..9d45532eec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + } + + @Override + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + callback.onSuccess(); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { + callback.onSuccess(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index b2e33e1705..86556d7adc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -16,8 +16,10 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -32,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldState import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -44,12 +47,12 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${service.type:null}'=='monolith'") -public class RocksDBStateService implements CalculatedFieldStateService { +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { + private final ActorSystemContext actorSystemContext; private final RocksDBService rocksDBService; - @Override public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( @@ -58,6 +61,12 @@ public class RocksDBStateService implements CalculatedFieldStateService { )); } + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index fa02435ca0..2a090fe701 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -674,7 +674,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); @@ -687,7 +687,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getDeviceProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); @@ -710,7 +710,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() @@ -720,7 +720,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getAssetProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -872,6 +872,6 @@ public class DefaultTbClusterService implements TbClusterService { private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } From cfa7066ac62b602d32470ae61ae548604067a15e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:23:38 +0200 Subject: [PATCH 149/281] Resolved comments --- ...culated-field-arguments-table.component.ts | 2 +- ...ulated-field-argument-panel.component.html | 4 ++-- ...ulated-field-argument-panel.component.scss | 22 ------------------- ...lculated-field-argument-panel.component.ts | 13 +++++------ 4 files changed, 9 insertions(+), 32 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 171c382b7a..3c51dc7f79 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -143,7 +143,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, - argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index a4f05b3288..4e9819a786 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -111,7 +111,7 @@

} @else { - @if (isDeviceEntity) { + @if (enableAttributeScopeSelection) {
{{ 'calculated-fields.attribute-scope' | translate }}
@@ -155,7 +155,7 @@
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss deleted file mode 100644 index 45c17628d5..0000000000 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host ::ng-deep { - .time-window-field { - .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { - border-left: 1px solid rgba(0, 0, 0, 0) !important; - } - } -} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 1632bd9311..a2e5545926 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; +import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,7 +42,6 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', - styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -53,7 +52,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; - @Input() argumentNames: string[]; + @Input() usedArgumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -111,7 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } - get isDeviceEntity(): boolean { + get enableAttributeScopeSelection(): boolean { return this.entityType === ArgumentEntityType.Device || (this.entityType === ArgumentEntityType.Current && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) @@ -196,16 +195,16 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); - if (!this.isDeviceEntity) { + if (!this.enableAttributeScopeSelection) { this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); } }); } private uniqNameRequired(): ValidatorFn { - return (control: UntypedFormControl) => { + return (control: FormControl) => { const newName = control.value.trim().toLowerCase(); - const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); return isDuplicate ? { duplicateName: true } : null; }; From 0e1c3d66ff9a27cb00cfa3c7e0601a5723282468 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:29:27 +0200 Subject: [PATCH 150/281] Added tb-form-panel-title tb-required --- .../dialog/calculated-field-dialog.component.html | 4 ++-- ui-ngx/src/form.scss | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d708f060a..8d69bf6539 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
-
{{ 'calculated-fields.arguments' | translate }}*
+
{{ 'calculated-fields.arguments' | translate }}
-
{{ 'calculated-fields.expression' | translate }}*
+
{{ 'calculated-fields.expression' | translate }}
@if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index 0ae2b9584b..4c00a330c7 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -163,6 +163,13 @@ .tb-form-panel-title { font-weight: 500; font-size: 16px; + + &.tb-required::after { + font-size: 13px; + color: rgba(0, 0, 0, .54); + vertical-align: top; + content: " *"; + } } .tb-form-panel-hint { font-size: 12px; From c6f6408c22f6a724c6d7ed3ec1be68713e76ef05 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 17:04:01 +0200 Subject: [PATCH 151/281] refactored tests --- ...CalculatedFieldEntityMessageProcessor.java | 8 +- .../cf/CalculatedFieldIntegrationTest.java | 159 +++++++++--------- 2 files changed, 79 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 0d327adca9..00913ec66d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index fda0724b1b..f6dc09e3ce 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.cf; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; @@ -35,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -53,12 +54,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes @Test public void testSimpleCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - - JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); - - JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -72,15 +69,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); - config.setArguments(Map.of("T", argument)); - config.setExpression("(T * 9/5) + 32"); Output output = new Output(); output.setName("fahrenheitTemp"); output.setType(OutputType.TIME_SERIES); - config.setOutput(output); calculatedField.setConfiguration(config); @@ -91,19 +85,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); // update telemetry -> recalculate state - JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); Thread.sleep(300); - ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); - assertThat(fahrenheitTempAfterUpdate).isNotNull(); - assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -114,21 +107,20 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); // update CF argument -> perform calculation with new argument - Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); Thread.sleep(300); - ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateArg).isNotNull(); - assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); @@ -136,36 +128,23 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateExpression).isNotNull(); - assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); - Asset asset1 = new Asset(); - asset1.setName("Test asset 1"); - asset1.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); - JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); - - Asset asset2 = new Asset(); - asset2.setName("Test asset 2"); - asset2.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); - - JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(assetProfile.getId()); @@ -205,62 +184,74 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); // result of asset 1 - ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); assertThat(z1).isNotNull(); assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); // result of asset 2 - ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); // update device telemetry -> recalculate state for all assets - JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); Thread.sleep(300); // result of asset 1 - ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ1).isNotNull(); - assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); // result of asset 2 - ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ2).isNotNull(); - assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); - -// // update asset 1 telemetry -> recalculate state only for asset 1 -// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 (no changes) -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); -// -// // update asset 2 telemetry -> recalculate state only for asset 2 -// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 (no changes) -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 1 telemetry -> recalculate state only for asset 1 + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 (no changes) + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 2 telemetry -> recalculate state only for asset 2 + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); + + Thread.sleep(300); + + // result of asset 1 (no changes) + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); } } From 1256993569e75b0cb9c0fb72f04707b9d7900b7a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 10 Feb 2025 12:59:21 +0200 Subject: [PATCH 152/281] added new tests and fix proto serialization --- .../cf/CalculatedFieldIntegrationTest.java | 200 +++++++++++++++++- .../server/common/util/ProtoUtils.java | 4 +- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f6dc09e3ce..91aad7b1cd 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -52,7 +52,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedField() throws Exception { + public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); @@ -69,6 +69,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present config.setArguments(Map.of("T", argument)); config.setExpression("(T * 9/5) + 32"); @@ -133,6 +134,99 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> state is not ready -> no calculation performed + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> perform calculation + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation with default value + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + + // update telemetry -> recalculate state + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); @@ -237,6 +331,110 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + + // add new entity to profile -> calculate state for new entity + Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); + + Thread.sleep(300); + + // result of asset 3 + ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + + // update device telemetry -> recalculate state for all assets + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + + // result of asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + + // update profile for asset 3 -> delete state for asset 3 + AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); + asset3.setAssetProfileId(newAssetProfile.getId()); + asset3 = doPost("/api/asset", asset3, Asset.class); + + // update device telemetry -> recalculate state for asset 1 and asset 2 + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/0) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> ctx is not initialized -> no calculation perform + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> ctx is not initialized -> no calculation perform + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index b81af2bfb9..07406977d2 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -127,8 +127,8 @@ public class ProtoUtils { builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); } if (msg.getOldProfileId() != null) { - builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); - builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } if (msg.getName() != null) { builder.setName(msg.getName()); From 55f279944660e2c2d92825ba799c52719e3d529c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 15:35:14 +0200 Subject: [PATCH 153/281] Implemented Calculated Fields debug test --- .../core/http/calculated-fields.service.ts | 5 + .../calculated-fields-table-config.ts | 34 +++- ...lated-field-arguments-table.component.html | 2 +- ...lculated-field-debug-dialog.component.html | 1 + ...calculated-field-debug-dialog.component.ts | 6 +- .../calculated-field-dialog.component.html | 8 + .../calculated-field-dialog.component.ts | 12 +- .../components/public-api.ts | 1 + ...ulated-field-test-arguments.component.html | 36 ++++ ...lculated-field-test-arguments.component.ts | 88 ++++++++++ ...ed-field-script-test-dialog.component.html | 101 ++++++++++++ ...ed-field-script-test-dialog.component.scss | 51 ++++++ ...ated-field-script-test-dialog.component.ts | 154 ++++++++++++++++++ .../components/event/event-table-config.ts | 10 ++ .../home/components/home-components.module.ts | 10 ++ .../shared/models/calculated-field.models.ts | 12 ++ ui-ngx/src/app/shared/models/entity.models.ts | 2 +- .../assets/locale/locale.constant-en_US.json | 6 + 18 files changed, 527 insertions(+), 12 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 9d8658f124..8e7ab6795e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,6 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; +import { TestScriptResult } from '@shared/models/rule-node.models'; @Injectable({ providedIn: 'root' @@ -48,4 +49,8 @@ export class CalculatedFieldsService { return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public testScript(inputParams: any, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 026c249159..8c00cbc925 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,9 +38,13 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData + CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; -import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; +import { + CalculatedFieldDebugDialogComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent +} from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; @@ -53,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), + action: (id?: CalculatedFieldId, expression?: string) => this.openDebugDialog.call(this, id, expression), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -134,10 +138,10 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id) + action: () => this.openDebugDialog(id, configuration?.expression) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -198,19 +202,22 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, entityId: this.entityId, - id + id, + expression, + testScriptFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -251,4 +258,17 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } + + private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: argumentsObj, + expression, + withApply, + } + }).afterClosed(); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index d8a6c7cdb0..5ca80b8214 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -24,7 +24,7 @@

{{ 'entity.key' | translate }}
-
+
@for (group of argumentsFormArray.controls; track group) {
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 91da675fea..f88176be8a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -34,6 +34,7 @@ [active]="true" [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" + (debugEventSelected)="onDebugEventSelected($event)" />
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index d81295948d..5b79476528 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -20,7 +20,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { DebugEventType, EventType } from '@shared/models/event.models'; +import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; @@ -51,4 +51,8 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent +
+ +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d5bf243430..0613ccf1b7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { filter, map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -67,7 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action(this.data.value.id) + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id, this.data.value.configuration.expression) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; @@ -110,6 +110,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), + this.configFormGroup.get('expressionSCRIPT').value, + true + ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + } + private applyDialogData(): void { const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value ?? {}; const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index 78b8862c2e..14ae73f7e1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -18,3 +18,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; export * from './debug-dialog/calculated-field-debug-dialog.component'; +export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html new file mode 100644 index 0000000000..e6b62a6862 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -0,0 +1,36 @@ + +
+
{{ 'calculated-fields.arguments' | translate }}
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'common.value' | translate }}
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + +
+ } +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts new file mode 100644 index 0000000000..d7a33d1a89 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -0,0 +1,88 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef } from '@angular/core'; +import { + ControlValueAccessor, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + ValidationErrors, + FormBuilder, + FormGroup +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +@Component({ + selector: 'tb-calculated-field-test-arguments', + templateUrl: './calculated-field-test-arguments.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true, + } + ] +}) +export class CalculatedFieldTestArgumentsComponent extends PageComponent implements ControlValueAccessor, Validator { + + + argumentsFormArray = this.fb.array([]); + + private propagateChange: (value: { argumentName: string; value: unknown }) => void; + + constructor(private fb: FormBuilder) { + super(); + this.argumentsFormArray.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(() => this.propagateChange(this.getValue())); + } + + registerOnChange(propagateChange: (value: { argumentName: string; value: unknown }) => void): void { + this.propagateChange = propagateChange; + } + + registerOnTouched(_): void { + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [{ value: key, disabled: true}], + value: [argumentsObj[key]] + }) as FormGroup, {emitEvent: false}); + }); + } + + validate(): ValidationErrors | null { + return this.argumentsFormArray.valid ? null : { arguments: { valid: false } }; + } + + private getValue(): { argumentName: string; value: unknown } { + return this.argumentsFormArray.getRawValue().reduce((acc, rowItem) => { + const { argumentName, value } = rowItem; + acc[argumentName] = value; + return acc; + }, {}) as { argumentName: string; value: unknown }; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html new file mode 100644 index 0000000000..ecfe2d41aa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -0,0 +1,101 @@ + +
+ +

{{ 'calculated-fields.test-script-function' | translate }} ({{ 'TBEL' }})

+ + +
+
+
+
+
+
+ {{ 'calculated-fields.expression' | translate }} +
+ +
+
+
+
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+ +
+
+
+
+
+ common.output +
+ +
+
+
+
+
+
+
+ + + + @if (data.withApply) { + + } +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss new file mode 100644 index 0000000000..ee0d59b839 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .test-dialog-container { + .block-label { + padding: 4px; + color: #00acc1; + background: rgba(220, 220, 220, .35); + border-radius: 5px; + } + + .test-block-content { + padding-top: 5px; + padding-left: 5px; + border: 1px solid #c0c0c0; + } + } +} + +:host::ng-deep { + .test-dialog-container { + .gutter { + background-color: #eee; + background-repeat: no-repeat; + background-position: 50%; + } + + .gutter.gutter-horizontal { + cursor: col-resize; + background-image: url("../../../../../../../assets/split.js/grips/horizontal.png"); + } + + .gutter.gutter-vertical { + cursor: row-resize; + background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts new file mode 100644 index 0000000000..63ee6fc5a7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + DestroyRef, + ElementRef, + Inject, + ViewChild, +} from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder } from '@angular/forms'; +import { NEVER, Observable, of, switchMap } from 'rxjs'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { ContentType } from '@shared/models/constants'; +import { JsonContentComponent } from '@shared/components/json-content.component'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { beautifyJs } from '@shared/models/beautify.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-script-test-dialog', + templateUrl: './calculated-field-script-test-dialog.component.html', + styleUrls: ['./calculated-field-script-test-dialog.component.scss'], +}) +export class CalculatedFieldScriptTestDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild('leftPanel', {static: true}) leftPanelElmRef: ElementRef; + @ViewChild('rightPanel', {static: true}) rightPanelElmRef: ElementRef; + @ViewChild('topRightPanel', {static: true}) topRightPanelElmRef: ElementRef; + @ViewChild('bottomRightPanel', {static: true}) bottomRightPanelElmRef: ElementRef; + + @ViewChild('expressionContent', {static: true}) expressionContent: JsonContentComponent; + + calculatedFieldScriptTestFormGroup = this.fb.group({ + expression: [], + arguments: [], + output: [] + }); + + readonly ContentType = ContentType; + readonly ScriptLanguage = ScriptLanguage; + readonly functionArgs = Object.keys(this.data.arguments); + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + protected dialogRef: MatDialogRef, + private fb: FormBuilder, + private destroyRef: DestroyRef, + private calculatedFieldService: CalculatedFieldsService) { + super(store, router, dialogRef); + beautifyJs(this.data.expression, {indent_size: 4}).pipe(takeUntilDestroyed()).subscribe( + (res) => { + this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); + } + ); + this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); + } + + ngAfterViewInit(): void { + this.initSplitLayout( + this.leftPanelElmRef.nativeElement, + this.rightPanelElmRef.nativeElement, + this.topRightPanelElmRef.nativeElement, + this.bottomRightPanelElmRef.nativeElement + ); + } + + cancel(): void { + this.dialogRef.close(null); + } + + onTestScript(): void { + this.testScript() + .pipe( + switchMap(output => beautifyJs(output, {indent_size: 4})), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe(output => this.calculatedFieldScriptTestFormGroup.get('output').setValue(output)); + } + + save(): void { + this.testScript().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); + this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); + }); + } + + private testScript(): Observable { + if (this.checkInputParamErrors()) { + return this.calculatedFieldService.testScript({ + expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, + arguments: this.calculatedFieldScriptTestFormGroup.get('arguments').value + }).pipe( + switchMap(result => { + if (result.error) { + this.store.dispatch(new ActionNotificationShow( + { + message: result.error, + type: 'error' + })); + return NEVER; + } else { + return of(result.output); + } + }), + ); + } else { + return NEVER; + } + } + + private checkInputParamErrors(): boolean { + this.expressionContent.validateOnSubmit(); + return !this.calculatedFieldScriptTestFormGroup.get('expression').invalid; + } + + private initSplitLayout(leftPanel, rightPanel, topRightPanel, bottomRightPanel): void { + Split([leftPanel, rightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'col-resize' + }); + + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a937e42bb8..c207e2d161 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -457,6 +457,16 @@ export class EventTableConfig extends EntityTableConfig { }); } break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.cellActionDescriptors.push({ + name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), + icon: 'bug_report', + isEnabled: () => true, + onAction: (_, entity) => { + this.debugEventSelected.next(entity.body); + } + }); + break; } this.getTable()?.cellActionDescriptorsUpdated(); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index b7e35c5406..ea8bf7acf8 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -198,6 +198,12 @@ import { import { CalculatedFieldDebugDialogComponent } from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; @NgModule({ declarations: @@ -347,6 +353,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], imports: [ CommonModule, @@ -490,6 +498,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 31dca236a4..e1f3ed8983 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -26,6 +26,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType } from '@shared/models/alias.models'; +import { Observable } from 'rxjs'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId, ExportableEntity { debugSettings?: EntityDebugSettings; @@ -126,6 +127,8 @@ export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { argumentName: string; } +export type CalculatedFieldTestScriptFn = (argumentsObj: Record, expression: string, withApply?: boolean) => Observable; + export interface CalculatedFieldDialogData { value?: CalculatedField; buttonTitle: string; @@ -134,12 +137,21 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig; + testScriptFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldDebugDialogData { id?: CalculatedFieldId; entityId: EntityId; tenantId: string; + expression?: string; + testScriptFn: CalculatedFieldTestScriptFn; +} + +export interface CalculatedFieldScriptTestDialogData { + arguments: Record, + expression: string; + withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index b9d2402850..1f73c0c8eb 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -204,7 +204,7 @@ export interface EntityDebugSettings { } export interface AdditionalDebugActionConfig { - action?: (id?: EntityId) => void; + action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d17409e52d..4822d23387 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -65,6 +65,7 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", + "test": "Test", "done": "Done", "print": "Print", "restore": "Restore", @@ -1018,6 +1019,7 @@ "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", + "test-script-function": "Test script function", "no-arguments": "No arguments configured", "argument-settings": "Argument settings", "argument-current": "Current entity", @@ -1107,8 +1109,12 @@ "proceed": "Proceed", "open-details-page": "Open details page", "not-found": "Not found", + "value": "Value", "documentation": "Documentation", "time-left": "{{time}} left", + "output": "Output", + "test-function": "Test function", + "test-with-this-message": "{{test}} with this message", "suffix": { "s": "s", "ms": "ms" From 5efb94dc7ae8f69d1d8e156f42ddb08fb72ca4f0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 10 Feb 2025 15:58:01 +0200 Subject: [PATCH 154/281] Refactoring --- .../server/actors/ActorSystemContext.java | 6 +- .../server/actors/app/AppActor.java | 1 + .../CalculatedFieldEntityActor.java | 4 + ...CalculatedFieldEntityMessageProcessor.java | 20 +- .../CalculatedFieldManagerActor.java | 5 +- ...alculatedFieldManagerMessageProcessor.java | 15 +- .../server/actors/tenant/TenantActor.java | 1 + ... => CalculatedFieldProcessingService.java} | 16 +- .../cf/CalculatedFieldQueueService.java | 37 +++ .../CalculatedFieldStateService.java | 3 +- .../cf/DefaultCalculatedFieldInitService.java | 2 - ...aultCalculatedFieldProcessingService.java} | 287 +----------------- .../DefaultCalculatedFieldQueueService.java | 238 +++++++++++++++ .../CalculatedFieldEntityProfileCache.java | 1 + ...aultCalculatedFieldEntityProfileCache.java | 9 +- .../KafkaCalculatedFieldStateService.java | 25 +- .../RocksDBCalculatedFieldStateService.java | 2 +- ...faultTbCalculatedFieldConsumerService.java | 22 +- .../queue/DefaultTbClusterService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 12 +- .../server/common/msg/MsgType.java | 2 + .../cf/CalculatedFieldPartitionChangeMsg.java | 40 +++ .../queue/discovery/HashPartitionService.java | 5 + .../queue/discovery/PartitionService.java | 2 + 24 files changed, 408 insertions(+), 351 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/{CalculatedFieldExecutionService.java => CalculatedFieldProcessingService.java} (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java rename application/src/main/java/org/thingsboard/server/service/cf/{ctx => }/CalculatedFieldStateService.java (90%) rename application/src/main/java/org/thingsboard/server/service/cf/{DefaultCalculatedFieldExecutionService.java => DefaultCalculatedFieldProcessingService.java} (55%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c714ed7dab..c787fc26af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,9 +106,9 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -526,7 +526,7 @@ public class ActorSystemContext { @Lazy @Autowired(required = false) @Getter - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; @Lazy @Autowired(required = false) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 8a6907c107..8c70eaf617 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -88,6 +88,7 @@ public class AppActor extends ContextAwareActor { case APP_INIT_MSG: break; case PARTITION_CHANGE_MSG: + case CF_PARTITIONS_CHANGE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cebe6b6a60..5e4aded41f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @Slf4j public class CalculatedFieldEntityActor extends ContextAwareActor { @@ -50,6 +51,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.process((CalculatedFieldPartitionChangeMsg) msg); + break; case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 00913ec66d..61f7abb297 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -30,15 +30,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,8 +68,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; - final CalculatedFieldExecutionService cfService; + final CalculatedFieldProcessingService cfService; final CalculatedFieldStateService cfStateService; + final int partition; TbActorCtx ctx; Map states = new HashMap<>(); @@ -77,14 +79,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM super(systemContext); this.tenantId = tenantId; this.entityId = entityId; - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); } void init(TbActorCtx ctx) { this.ctx = ctx; } + public void process(CalculatedFieldPartitionChangeMsg msg) { + if (!msg.getPartitions()[partition]) { + log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); + ctx.stop(ctx.getSelf()); + } + } + public void process(CalculatedFieldStateRestoreMsg msg) { log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); @@ -202,7 +212,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 22909dd5af..497602c93b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; /** * Created by ashvayka on 15.03.18. @@ -55,8 +56,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { - case PARTITION_CHANGE_MSG: - ctx.broadcastToChildren(msg, true); // TODO + case CF_PARTITIONS_CHANGE_MSG: + processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index cdc31ed93c..22492f3c4f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -35,14 +35,15 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -52,6 +53,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,20 +70,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + protected final TenantId tenantId; protected TbActorCtx ctx; - final TenantId tenantId; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); - this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); @@ -478,4 +480,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } + public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + ctx.broadcastToChildren(msg, true); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 1b008f462a..16ea2c1398 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,6 +170,7 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: + case CF_PARTITIONS_CHANGE_MSG: case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 47fd560b4d..24b428593a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -15,15 +15,11 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -31,17 +27,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; -public interface CalculatedFieldExecutionService { - - /** - * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * - * @param request - telemetry save request; - * @param callback - */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - - void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); +public interface CalculatedFieldProcessingService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java new file mode 100644 index 0000000000..b84b54af81 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; + +import java.util.List; + +public interface CalculatedFieldQueueService { + + /** + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * + * @param request - telemetry save request; + * @param callback + */ + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index ce1562c735..37211c66c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cf.ctx; +package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 2afd6d8238..f8eb7e852e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -20,8 +20,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e84e785a94..6b47053f2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -32,25 +31,17 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -59,7 +50,6 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -67,12 +57,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; @@ -80,12 +67,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinke import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -93,149 +79,51 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; -import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; +@TbRuleEngineComponent @Service @Slf4j @RequiredArgsConstructor -public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { +public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { - public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - } - - @Override - public void onFailure(Throwable t) { - } - }; - - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; + private final PartitionService partitionService; - private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - private static final Set supportedReferencedEntities = EnumSet.of( - EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT - ); - @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @PostConstruct public void init() { - super.init(); - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); } @PreDestroy public void stop() { - super.stop(); - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } if (calculatedFieldCallbackExecutor != null) { calculatedFieldCallbackExecutor.shutdownNow(); } } - @Override - protected String getServiceName() { - return "Calculated Field Execution"; - } - - @Override - protected String getSchedulerExecutorName() { - return "calculated-field-scheduled"; - } - - @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; - //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; - //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, - Predicate mainEntityFilter, Predicate linkedEntityFilter, - Supplier msg, FutureCallback callback) { - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); - if (send) { - clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); - } else { - if (callback != null) { - callback.onSuccess(null); - } - } - } - - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { - boolean send = false; - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); - if (!send) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); - } - if (!send) { - send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) - .map(calculatedFieldCache::getCalculatedFieldCtx) - .anyMatch(linkedEntityFilter); - } - } - return send; - } - @Override public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { Map> argFutures = new HashMap<>(); @@ -262,21 +150,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - protected Map>> onAddedPartitions(Set addedPartitions) { - var result = new HashMap>>(); - return result; - } - - @Override - protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - cleanupEntity(entityId); - } - - private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - } - @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { @@ -369,30 +242,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); @@ -462,78 +311,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private boolean isProfileEntity(EntityId entityId) { - return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); - } - - private EntityId getProfileId(TenantId tenantId, EntityId entityId) { - return switch (entityId.getEntityType()) { - case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> null; - }; - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - List entries = request.getEntries(); - List versions = result.getVersions(); - for (int i = 0; i < entries.size(); i++) { - long tsVersion = versions.get(i); - TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); - telemetryMsg.addTsData(tsProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - List entries = request.getEntries(); - for (int i = 0; i < entries.size(); i++) { - long attrVersion = versions.get(i); - AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); - telemetryMsg.addAttrData(attrProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); - - telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - - telemetryMsg.setEntityType(entityId.getEntityType().name()); - telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (calculatedFieldIds != null) { - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); - } - } - - if (tbMsgId != null) { - telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); - telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); - } - - if (tbMsgType != null) { - telemetryMsg.setTbMsgType(tbMsgType.name()); - } - - return telemetryMsg; - } - private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -541,60 +318,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } else { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, scope, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - } - - private static TbQueueCallback wrap(FutureCallback callback) { - if (callback != null) { - return new FutureCallbackWrapper(callback); - } else { - return DUMMY_TB_QUEUE_CALLBACK; - } - } - - private static class FutureCallbackWrapper implements TbQueueCallback { - private final FutureCallback callback; - - public FutureCallbackWrapper(FutureCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - callback.onSuccess(null); - } - - @Override - public void onFailure(Throwable t) { - callback.onFailure(t); - } - } - private static class TbCallbackWrapper implements TbQueueCallback { private final TbCallback callback; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java new file mode 100644 index 0000000000..9d1f9b6db5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { + + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; + private final TbClusterService clusterService; + + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } + } + + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + + return telemetryMsg; + } + + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java index 6b0718eb84..fec8c8ba12 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -32,4 +32,5 @@ public interface CalculatedFieldEntityProfileCache extends ApplicationListener

getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + int getEntityIdPartition(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 866cc86f24..9a56d35ac4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -42,7 +42,7 @@ import java.util.stream.Collectors; //TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { - private static final Integer UNKNOWN = -1; + private static final Integer UNKNOWN = 0; private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); private final PartitionService partitionService; private volatile List myPartitions = Collections.emptyList(); @@ -84,4 +84,11 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); } + + @Override + public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + return tpi.getPartition().orElse(UNKNOWN); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 9d45532eec..a1999d438c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -17,34 +17,11 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; - -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; @Service @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 86556d7adc..48e17954ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -37,7 +37,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import java.util.Map; import java.util.Optional; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 3d8bbddc80..f3e976084b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -32,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -49,7 +52,6 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -58,6 +60,7 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -134,8 +137,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + var partitions = event.getCalculatedFieldsPartitions(); + log.info("Subscribing to partitions: {}", partitions); + // TODO: @vklimov - before update of the main consumer, we should read the state topics and use + // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. + // Cleanup entities that do not belong to current partition; mainConsumer.update(event.getCalculatedFieldsPartitions()); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } + + private boolean[] partitionsToBooleanIndexArray(Set partitions) { + boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; + for(var tpi : partitions) { + tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); + } + return myPartitions; } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2a090fe701..48d26895b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -96,7 +96,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,7 +148,7 @@ public class DefaultTbClusterService implements TbClusterService { @Autowired @Lazy - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 3f7f9c0b7d..b338a8382c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,7 +50,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,7 +77,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final CalculatedFieldQueueService calculatedFieldQueueService; private ExecutorService tsCallBackExecutor; @@ -89,13 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldQueueService calculatedFieldQueueService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.calculatedFieldQueueService = calculatedFieldQueueService; } @PostConstruct @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index e13e2d2ecb..9e5f00abf2 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -139,6 +139,8 @@ public enum MsgType { CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; CF_TELEMETRY_MSG, // Sent from queue to actor system; CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java new file mode 100644 index 0000000000..5ab4b0ad9f --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +import java.util.Set; + +@Data +public class CalculatedFieldPartitionChangeMsg implements ToCalculatedFieldSystemMsg { + + private final boolean[] partitions; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_PARTITIONS_CHANGE_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 5c6bf545b5..f2f1ccd19c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -526,6 +526,11 @@ public class HashPartitionService implements PartitionService { return list == null ? 0 : list.size(); } + @Override + public int getTotalCalculatedFieldPartitions() { + return cfPartitions; + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 8e2152830e..b0e1229f97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -77,4 +77,6 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int getTotalCalculatedFieldPartitions(); + } From 84b9bde5772020787e4e842d1fd5d6971e35fc7d Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 16:11:46 +0200 Subject: [PATCH 155/281] Refactoring --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 6 +++--- ui-ngx/src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields/calculated-fields-table-config.ts | 3 ++- .../calculated-field-debug-dialog.component.scss | 1 - .../dialog/calculated-field-dialog.component.ts | 5 ++++- .../calculated-field-script-test-dialog.component.html | 4 ++-- .../calculated-field-script-test-dialog.component.ts | 7 +++---- .../modules/home/components/event/event-table-config.ts | 8 +++----- ui-ngx/src/app/shared/models/calculated-field.models.ts | 7 +++++-- ui-ngx/src/app/shared/models/entity.models.ts | 5 +++++ ui-ngx/src/app/shared/models/rule-node.models.ts | 5 ----- 11 files changed, 28 insertions(+), 25 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 8e7ab6795e..acaf3b2817 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,10 +19,10 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/rule-node.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: any, config?: RequestConfig): Observable { + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index e3353989cc..c1333df1ac 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,7 +35,6 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, - TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -44,6 +43,7 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 8c00cbc925..c71eef8de3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,7 +38,8 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData + CalculatedFieldDialogData, + CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 23aa4070d7..19bf072b11 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,6 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 80vw; .debug-dialog-content { border-radius: 0; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 0613ccf1b7..20a37c8cc3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -115,7 +115,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), this.configFormGroup.get('expressionSCRIPT').value, true - ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + ).pipe(filter(Boolean)).subscribe((expression: string) => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index ecfe2d41aa..7af5e55b1c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -48,10 +48,10 @@

-
+
{{ 'calculated-fields.arguments' | translate }}
- +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index 63ee6fc5a7..fa66827eb6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,6 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; +import { filter } from 'rxjs/operators'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -71,10 +72,8 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { - this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); - } + beautifyJs(this.data.expression, {indent_size: 4}).pipe(filter(Boolean), takeUntilDestroyed()).subscribe( + (res) => this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}) ); this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index c207e2d161..39fb143de0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 6)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 6)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -462,9 +462,7 @@ export class EventTableConfig extends EntityTableConfig { name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), icon: 'bug_report', isEnabled: () => true, - onAction: (_, entity) => { - this.debugEventSelected.next(entity.body); - } + onAction: (_, entity) => this.debugEventSelected.next(entity.body) }); break; } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index e1f3ed8983..1a208e122f 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -148,10 +148,13 @@ export interface CalculatedFieldDebugDialogData { testScriptFn: CalculatedFieldTestScriptFn; } -export interface CalculatedFieldScriptTestDialogData { +export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { + withApply: boolean; +} + +export interface CalculatedFieldTestScriptInputParams { arguments: Record, expression: string; - withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 1f73c0c8eb..3a8c06f544 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,6 +203,11 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export interface TestScriptResult { + output: string; + error: string; +} + export interface AdditionalDebugActionConfig { action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index a79fdbc376..ac08fa38ac 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -374,11 +374,6 @@ export interface TestScriptInputParams { msgType: string; } -export interface TestScriptResult { - output: string; - error: string; -} - export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', From d75bc3ac28e52ec44ef88ad5f238cb3cfac46cd9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 11 Feb 2025 12:47:37 +0200 Subject: [PATCH 156/281] Minor improvements --- .../cf/ctx/state/RocksDBCalculatedFieldStateService.java | 5 ++--- .../java/org/thingsboard/server/queue/util/AfterStartUp.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 48e17954ea..3374eb2660 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -47,7 +47,7 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { private final ActorSystemContext actorSystemContext; @@ -66,12 +66,11 @@ public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateS restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); } - @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index f97c3b0a1b..dea6d9ed9e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -41,7 +41,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; int CF_STATE_RESTORE_SERVICE = 12; - int CF_CONSUMER_SERVICE = 13; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From 9e19fab11dda76ebf65c88ac77c311ee175b6bc2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:43:13 +0200 Subject: [PATCH 157/281] Review comments resolving and refactoring --- .../core/http/calculated-fields.service.ts | 6 +-- .../src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields-table-config.ts | 44 ++++++++++--------- ...lculated-field-debug-dialog.component.html | 2 +- ...calculated-field-debug-dialog.component.ts | 10 +++-- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.ts | 29 +++++++----- ...ed-field-script-test-dialog.component.html | 24 +++++----- ...ed-field-script-test-dialog.component.scss | 22 ++++++++++ ...ated-field-script-test-dialog.component.ts | 14 +++--- .../shared/models/calculated-field.models.ts | 15 +++---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +-- .../src/app/shared/models/rule-node.models.ts | 4 +- .../en_US/calculated-field/expression_fn.md | 1 + .../calculated-field/test-expression_fn.md | 1 + 15 files changed, 109 insertions(+), 73 deletions(-) create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index acaf3b2817..3e0e08f8e6 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,7 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/entity.models'; +import { EntityTestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { - return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index c1333df1ac..e3353989cc 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,6 +35,7 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, + TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -43,7 +44,6 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; -import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c71eef8de3..63b0fad732 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -34,12 +34,12 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, filter, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldScriptTestDialogData + CalculatedFieldTestScriptInputParams, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -47,7 +47,6 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; -import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id, expression), + action: (calculatedField: CalculatedField) => this.openDebugDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -139,10 +138,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id, configuration?.expression) + action: () => this.openDebugDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -178,8 +178,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), @@ -191,7 +191,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add', isDirty = false): Observable { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -204,21 +204,20 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, - entityId: this.entityId, - id, - expression, - testScriptFn: this.getTestScriptDialog.bind(this), + value: calculatedField, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -260,16 +259,21 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj, - expression, - withApply, + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => ({...acc, [key]: '' }), {}), + expression: calculatedField.configuration.expression, } - }).afterClosed(); + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + ), + ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index f88176be8a..8295a9c892 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data.id" + [entityId]="data.value.id" [functionTestButtonLabel]="'common.test-function' | translate" (debugEventSelected)="onDebugEventSelected($event)" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index 5b79476528..be27e277a6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -22,14 +22,14 @@ import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDebugDialogData, CalculatedFieldType } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-debug-dialog', styleUrls: ['calculated-field-debug-dialog.component.scss'], templateUrl: './calculated-field-debug-dialog.component.html', }) -export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; @@ -40,12 +40,13 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, - protected dialogRef: MatDialogRef) { + protected dialogRef: MatDialogRef) { super(store, router, dialogRef); } ngAfterViewInit(): void { this.eventsTable.entitiesTable.updateData(); + this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = () => this.data.value.type === CalculatedFieldType.SCRIPT; } cancel(): void { @@ -53,6 +54,7 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent this.dialogRef.close(expression)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ad2d14cbbf..f50c552ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,7 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: [Calculated Fields] add valid link" + helpId="calculated-field/expression_fn" />
- @if (data.withApply) { - - } +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index ee0d59b839..2420aa5a50 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -27,6 +27,28 @@ padding-left: 5px; border: 1px solid #c0c0c0; } + + .block-label-container { + position: absolute; + z-index: 10; + font-size: 12px; + font-weight: bold; + + &.left { + right: 112px; + top: 9px; + } + + &.right-bottom { + right: 40px; + top: 6px; + } + + &.right-top { + right: 8px; + top: 2px; + } + } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index fa66827eb6..bd19a68045 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -22,7 +22,7 @@ import { Inject, ViewChild, } from '@angular/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder } from '@angular/forms'; @@ -36,8 +36,8 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { filter } from 'rxjs/operators'; +import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,8 +66,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, protected dialogRef: MatDialogRef, + private dialog: MatDialog, private fb: FormBuilder, private destroyRef: DestroyRef, private calculatedFieldService: CalculatedFieldsService) { @@ -101,13 +102,13 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { + this.testScript(true).pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); }); } - private testScript(): Observable { + private testScript(onSave = false): Observable { if (this.checkInputParamErrors()) { return this.calculatedFieldService.testScript({ expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, @@ -122,6 +123,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, expression: string, withApply?: boolean) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -136,20 +136,15 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; - additionalDebugActionConfig: AdditionalDebugActionConfig; + additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; testScriptFn: CalculatedFieldTestScriptFn; + isDirty?: boolean; } export interface CalculatedFieldDebugDialogData { - id?: CalculatedFieldId; - entityId: EntityId; tenantId: string; - expression?: string; - testScriptFn: CalculatedFieldTestScriptFn; -} - -export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { - withApply: boolean; + value: CalculatedField; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldTestScriptInputParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 3a8c06f544..472d28f849 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,13 +203,13 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export interface TestScriptResult { +export interface EntityTestScriptResult { output: string; error: string; } -export interface AdditionalDebugActionConfig { - action?: (id?: EntityId, ...restArguments: unknown[]) => void; +export interface AdditionalDebugActionConfig void> { + action?: Action; title: string; } diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index ac08fa38ac..3e1eb70835 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -27,7 +27,7 @@ import { AppState } from '@core/core.state'; import { AbstractControl, UntypedFormGroup } from '@angular/forms'; import { RuleChainType } from '@shared/models/rule-chain.models'; import { DebugRuleNodeEventBody } from '@shared/models/event.models'; -import { HasEntityDebugSettings } from '@shared/models/entity.models'; +import { EntityTestScriptResult, HasEntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; export interface RuleNodeConfiguration { @@ -374,6 +374,8 @@ export interface TestScriptInputParams { msgType: string; } +export type TestScriptResult = EntityTestScriptResult; + export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md @@ -0,0 +1 @@ + From 2833164968252d1011a3d64d53dbb3d73650c106 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:48:45 +0200 Subject: [PATCH 158/281] Refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 2 +- .../components/dialog/calculated-field-dialog.component.ts | 2 +- ui-ngx/src/app/shared/models/calculated-field.models.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 63b0fad732..544860cca2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -203,7 +203,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.configFormGroup.get('expressionSCRIPT').setValue(expression); this.configFormGroup.get('expressionSCRIPT').markAsDirty(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index cec8fe5cd6..19b53bc91e 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -137,7 +137,7 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; - testScriptFn: CalculatedFieldTestScriptFn; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; isDirty?: boolean; } From f8d88387fa78bb7f8e93efbe759f8e9a40cad9d3 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:04:23 +0200 Subject: [PATCH 159/281] refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 5 +++-- .../dialog/calculated-field-dialog.component.ts | 8 ++------ .../modules/home/components/event/event-table-config.ts | 4 ++-- ui-ngx/src/app/shared/models/entity.models.ts | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 544860cca2..318ffaca66 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -205,7 +205,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig ({...acc, [key]: '' }), {}), + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, } }).afterClosed() diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 8574ee99c2..04e1b539dd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { filter, map, startWith } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -121,11 +121,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + this.data.getTestScriptDialogFn(this.fromGroupValue).subscribe(); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 39fb143de0..98d2fbefaf 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 8)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 472d28f849..20a0ccbe77 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -209,7 +209,7 @@ export interface EntityTestScriptResult { } export interface AdditionalDebugActionConfig void> { - action?: Action; + action: Action; title: string; } From 9589317e9750bea3190798c8060deb522ada1e85 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:25:01 +0200 Subject: [PATCH 160/281] preserve edit dialog on test --- .../calculated-fields-table-config.ts | 15 +++++++++------ .../dialog/calculated-field-dialog.component.ts | 5 ++++- ...lculated-field-script-test-dialog.component.ts | 6 +++--- .../app/shared/models/calculated-field.models.ts | 6 +++++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 318ffaca66..966ac5f008 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,7 +39,7 @@ import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldTestScriptInputParams, + CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -260,21 +260,24 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, + openCalculatedFieldEdit } }).afterClosed() .pipe( filter(Boolean), - tap(expression => - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) - ), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + } + }), ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 04e1b539dd..e75c962268 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -121,7 +121,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index bd19a68045..94ffeeea52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,7 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { filter } from 'rxjs/operators'; -import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; +import { CalculatedFieldTestScriptDialogData } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,7 +66,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptDialogData, protected dialogRef: MatDialogRef, private dialog: MatDialog, private fb: FormBuilder, @@ -123,7 +123,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record, closeAllOnSave?: boolean) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -152,6 +152,10 @@ export interface CalculatedFieldTestScriptInputParams { expression: string; } +export interface CalculatedFieldTestScriptDialogData extends CalculatedFieldTestScriptInputParams { + openCalculatedFieldEdit?: boolean; +} + export interface ArgumentEntityTypeParams { title: string; entityType: EntityType From 6eeaecf88ff69c5ca5de32867dc29e0be6d1456e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:30:24 +0200 Subject: [PATCH 161/281] close debug panel on additional action --- .../entity/debug/entity-debug-settings-panel.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index dcc3355890..f7fd9f737b 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -54,7 +54,7 @@ } From 934a8daa4ea59b6132afcd48fd781f9812c512dc Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 17:29:44 +0200 Subject: [PATCH 162/281] only last arguments fix --- .../calculated-fields/calculated-fields-table-config.ts | 9 ++++++++- .../modules/home/components/event/event-table-config.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 966ac5f008..9560f264b5 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -261,12 +261,19 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { + const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); + const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { + if (emptyArguments.hasOwnProperty(key)) { + acc[key] = argumentsObj[key]; + } + return acc; + }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), + arguments: { ...emptyArguments, ...filledArguments }, expression: calculatedField.configuration.expression, openCalculatedFieldEdit } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 98d2fbefaf..64917b23c0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -357,7 +357,7 @@ export class EventTableConfig extends EntityTableConfig { break; case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '80px'; - this.columns[1].width = '20%'; + this.columns[1].width = '100px'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, From a1b9e941e5c11f9022479ea0e0ce52a58041df0c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 18:14:38 +0200 Subject: [PATCH 163/281] fixed debugging dialog height --- .../calculated-field-debug-dialog.component.html | 4 ++-- .../calculated-field-debug-dialog.component.scss | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 8295a9c892..2397162b05 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
Date: Tue, 11 Feb 2025 18:40:25 +0200 Subject: [PATCH 164/281] amde arguments not required --- .../calculated-field-test-arguments.component.html | 2 +- .../src/app/shared/components/value-input.component.html | 8 ++++---- ui-ngx/src/app/shared/components/value-input.component.ts | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index e6b62a6862..37f6ba4d27 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index e6138dd321..a7108f9174 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -32,7 +32,7 @@ - - -
- Date: Tue, 11 Feb 2025 19:06:05 +0200 Subject: [PATCH 165/281] refactoring --- .../calculated-fields-table-config.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 9560f264b5..ce47292397 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -47,6 +47,7 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { isObject } from '@core/utils'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -206,7 +207,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { - const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); - const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { - if (emptyArguments.hasOwnProperty(key)) { - acc[key] = argumentsObj[key]; - } + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, @@ -273,7 +271,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig Date: Tue, 11 Feb 2025 19:10:55 +0200 Subject: [PATCH 166/281] WIP: arguments --- .../CalculatedFieldTbelScriptEngine.java | 1 + .../ctx/state/ScriptCalculatedFieldState.java | 25 +++++++++- .../api/tbel/DefaultTbelInvokeService.java | 9 ++-- .../thingsboard/script/api/tbel/TbCfArg.java | 22 +++++++++ .../script/api/tbel/TbCfSingleValueArg.java | 30 ++++++++++++ .../script/api/tbel/TbCfTsRollingArg.java | 47 +++++++++++++++++++ 6 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 9e05e05970..6dcbf33067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import io.jsonwebtoken.lang.Collections; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 4a24d13c93..7d86e376a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,12 +21,18 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.mvel2.execution.ExecutionArrayList; +import org.thingsboard.script.api.tbel.TbCfArg; +import org.thingsboard.script.api.tbel.TbCfSingleValueArg; +import org.thingsboard.script.api.tbel.TbCfTsRollingArg; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -62,8 +68,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { } }); Object[] args = ctx.getArgNames().stream() - .map(key -> arguments.get(key).getValue()) + .map(this::toTbelArgument) .toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, @@ -72,4 +79,20 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { ); } + private TbCfArg toTbelArgument(String key) { + ArgumentEntry argEntry = arguments.get(key); + if (argEntry instanceof SingleValueArgumentEntry svArg) { + return new TbCfSingleValueArg(svArg.getTs(), argEntry.getValue()); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArg) { + var tsRecords = rollingArg.getTsRecords(); + List values = new ArrayList<>(tsRecords.size()); + for(var e : tsRecords.entrySet()){ + values.add(new TbCfSingleValueArg(e.getKey(), e.getValue().getValue())); + } + return new TbCfTsRollingArg(values); + } else { + throw new RuntimeException("Argument is not supported for TBEL execution!"); + } + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 74d6a4e6f5..906e6ac393 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -32,6 +32,7 @@ import org.mvel2.MVEL; import org.mvel2.ParserContext; import org.mvel2.SandboxedParserConfiguration; import org.mvel2.ScriptMemoryOverflowException; +import org.mvel2.integration.PropertyHandlerFactory; import org.mvel2.optimizers.OptimizerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -130,9 +131,11 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); parserConfig = ParserContext.enableSandboxedMode(); parserConfig.addImport("JSON", TbJson.class); - parserConfig.registerDataType("Date", TbDate.class, date -> 8L); - parserConfig.registerDataType("Random", Random.class, date -> 8L); - parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + parserConfig.registerDataType("Date", TbDate.class, val -> 8L); + parserConfig.registerDataType("Random", Random.class, val -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, val -> 8L); + parserConfig.registerDataType("TbCfSingleValueArg", TbCfSingleValueArg.class, TbCfSingleValueArg::memorySize); + parserConfig.registerDataType("TbCfTsRollingArg", TbCfTsRollingArg.class, TbCfTsRollingArg::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java new file mode 100644 index 0000000000..f5d990a553 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +public interface TbCfArg { + + long memorySize(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java new file mode 100644 index 0000000000..e42fb64ba4 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbCfSingleValueArg implements TbCfArg { + + private final long ts; + private final Object value; + + @Override + public long memorySize() { + return 8L; // TODO; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java new file mode 100644 index 0000000000..a8abbc64ac --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Getter; + +import java.util.List; + +public class TbCfTsRollingArg implements TbCfArg { + + @Getter + private final List values; + + public TbCfTsRollingArg(List values) { + this.values = values; + } + + @Override + public long memorySize() { + return values.size() * 8L; //TODO; + } + + public double max() { + double max = Double.MIN_VALUE; + for (TbCfSingleValueArg arg : values) { + double val = Double.valueOf(arg.getValue().toString()); + if (max < val) { + max = val; + } + } + return max; + } + +} From e542267295c2818eb87f5f17b40deb16097b4f66 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 15:57:12 +0200 Subject: [PATCH 167/281] Added calculated field responsive styles --- ...lated-field-arguments-table.component.html | 18 ++--- ...lated-field-arguments-table.component.scss | 6 ++ ...lculated-field-debug-dialog.component.html | 4 +- ...lculated-field-debug-dialog.component.scss | 10 ++- .../calculated-field-dialog.component.html | 2 +- ...ulated-field-test-arguments.component.html | 4 +- ...ulated-field-test-arguments.component.scss | 29 ++++++++ ...lculated-field-test-arguments.component.ts | 1 + ...ed-field-script-test-dialog.component.html | 6 +- ...ated-field-script-test-dialog.component.ts | 73 ++++++++++++++----- .../components/event/event-table-config.ts | 8 +- 11 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 5ca80b8214..796fe7c138 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -18,19 +18,19 @@
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
@for (group of argumentsFormArray.controls; track group) {
- + -
+
@if (group.get('refEntityId')?.get('id')?.value) { @@ -63,7 +63,7 @@ }
- + @if (group.get('refEntityKey').get('type').value; as type) { @@ -72,9 +72,9 @@ } - + -
+
{{ group.get('refEntityKey').get('key').value }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index a0c90bba32..e6a48dee58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -27,4 +27,10 @@ font-size: 14px; } } + + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { + overflow: hidden; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 2397162b05..7da70e180e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
-
+

{{ 'entity.type-calculated-field' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 37f6ba4d27..03781f1ded 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -19,7 +19,7 @@
{{ 'calculated-fields.arguments' | translate }}
-
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.argument-name' | translate }}
{{ 'common.value' | translate }}
@@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss new file mode 100644 index 0000000000..1b2c8670c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host::ng-deep { + .tb-form-table-row { + .argument-value { + .tb-value-type.row { + @media #{constants.$mat-lt-sm} { + width: 100px; + min-width: 100px; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts index d7a33d1a89..c8c9f4e778 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -30,6 +30,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ selector: 'tb-calculated-field-test-arguments', templateUrl: './calculated-field-test-arguments.component.html', + styleUrls: ['./calculated-field-test-arguments.component.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index 03582b3066..3d3fe79e4b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.test-script-function' | translate }} ({{ 'api-usage.tbel' | translate }})

 -
+
{{ 'common.general' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 4e9819a786..8b3bb899ad 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{ 'calculated-fields.argument-settings' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..cfe7ee9461 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .fixed-title-width { + @media #{constants.$mat-lt-sm} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index a2e5545926..66ce0aeeeb 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 03781f1ded..08d023591a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index 2420aa5a50..eaee8e443d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -26,6 +26,7 @@ padding-top: 5px; padding-left: 5px; border: 1px solid #c0c0c0; + overflow: scroll; } .block-label-container { From 8b3fc83acc140ea33e415d741ba5c7fe5f9aed6c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:07:48 +0200 Subject: [PATCH 172/281] style fixes --- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.scss | 29 +++++++++++++++++++ .../calculated-field-dialog.component.ts | 1 + ...ulated-field-argument-panel.component.html | 2 +- ...ulated-field-argument-panel.component.scss | 7 +++++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 20245414dd..6ff0977178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'entity.type-calculated-field' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss new file mode 100644 index 0000000000..300193a6c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .dialog-container { + + @media #{constants.$mat-sm} { + min-width: 526px; + } + + @media #{constants.$mat-gt-sm} { + min-width: 770px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index e75c962268..31f07a1c0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -40,6 +40,7 @@ import { ScriptLanguage } from '@shared/models/rule-node.models'; @Component({ selector: 'tb-calculated-field-dialog', templateUrl: './calculated-field-dialog.component.html', + styleUrls: ['./calculated-field-dialog.component.scss'], }) export class CalculatedFieldDialogComponent extends DialogComponent implements AfterViewInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 8b3bb899ad..c2fe831204 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{ 'calculated-fields.argument-settings' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index cfe7ee9461..7c4e5c2f5e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -16,6 +16,13 @@ @use '../../../../../../../scss/constants' as constants; :host { + display: flex; + min-width: 520px; + + @media #{constants.$mat-lt-sm} { + min-width: 320px; + } + .fixed-title-width { @media #{constants.$mat-lt-sm} { min-width: 120px; From 9162acedee26f7ade43a6ad4dce9133de19b481b Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:19:06 +0200 Subject: [PATCH 173/281] style fixes --- .../panel/calculated-field-argument-panel.component.scss | 2 +- .../calculated-field-test-arguments.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index 7c4e5c2f5e..ca6d773262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -20,7 +20,7 @@ min-width: 520px; @media #{constants.$mat-lt-sm} { - min-width: 320px; + min-width: 280px; } .fixed-title-width { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 08d023591a..eea3523d00 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
From 1f28c0efe9b8a4f1d0c6f50d0c790dea6c2cfc27 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:40:41 +0200 Subject: [PATCH 174/281] style fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../dialog/calculated-field-dialog.component.scss | 10 ++-------- .../calculated-field-argument-panel.component.scss | 7 ++----- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 796fe7c138..77347c6477 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
@for (group of argumentsFormArray.controls; track group) { @@ -80,7 +80,7 @@ -
+
-
+
Date: Thu, 13 Feb 2025 12:33:30 +0200 Subject: [PATCH 180/281] fixed entity/message id min width --- .../app/modules/home/components/event/event-table-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 2f2650d434..b32c5f0050 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -359,7 +359,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '85px', + new EntityTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '85px', + new EntityTableColumn('messageId', 'event.message-id', '100px', (entity) => entity.body.msgId ? `${entity.body.msgId?.substring(0, 8)}…` : '-', () => ({padding: '0 12px 0 0'}), false, From 7232d939d86bbfe368cf955f08cd980efe3030f1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 12:57:26 +0200 Subject: [PATCH 181/281] fixed cf version --- .../server/dao/model/sql/CalculatedFieldEntity.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 64c8e8d5b8..55928dbed1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; @@ -52,7 +52,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = CALCULATED_FIELD_TABLE_NAME) -public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) private UUID tenantId; From 677385e8babc8f263e2b662c6142cd6311d40f4c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 13:01:17 +0200 Subject: [PATCH 182/281] CF states restore from Kafka --- ...CalculatedFieldEntityMessageProcessor.java | 9 +- .../AbstractCalculatedFieldStateService.java | 173 +++++++++++++++++ .../cf/CalculatedFieldStateService.java | 5 + .../cf/CfRocksDb.java} | 30 +-- ...faultCalculatedFieldProcessingService.java | 4 +- .../server/service/cf/RocksDBService.java | 97 ---------- ...aultCalculatedFieldEntityProfileCache.java | 11 +- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 + .../KafkaCalculatedFieldStateService.java | 129 ++++++++++++- .../RocksDBCalculatedFieldStateService.java | 175 +++--------------- ...faultTbCalculatedFieldConsumerService.java | 46 +++-- .../queue/DefaultTbClusterService.java | 9 +- .../DefaultTbRuleEngineConsumerService.java | 5 +- .../consumer/MainQueueConsumerManager.java | 10 +- .../queue/ruleengine/TbQueueConsumerTask.java | 12 +- .../auth/rest/RestAuthenticationProvider.java | 1 + .../thingsboard/server/utils/TbRocksDb.java | 71 +++++++ .../src/main/resources/thingsboard.yml | 8 +- .../server/common/data/DataConstants.java | 1 + .../common/data/util/CollectionsUtil.java | 12 ++ common/proto/src/main/proto/queue.proto | 13 +- .../AbstractTbQueueConsumerTemplate.java | 6 +- .../queue/discovery/HashPartitionService.java | 54 ++++-- .../queue/discovery/PartitionService.java | 2 + .../server/queue/discovery/QueueKey.java | 6 + .../discovery/event/PartitionChangeEvent.java | 6 +- .../server/queue/kafka/KafkaTbQueueMsg.java | 9 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 63 ++++++- .../queue/kafka/TbKafkaProducerTemplate.java | 7 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../InMemoryMonolithQueueFactory.java | 5 +- .../provider/KafkaMonolithQueueFactory.java | 40 ++-- .../KafkaTbRuleEngineQueueFactory.java | 40 ++-- .../provider/TbRuleEngineQueueFactory.java | 6 +- .../server/queue/util/AfterStartUp.java | 9 +- 35 files changed, 679 insertions(+), 405 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/{utils/RocksDBConfig.java => service/cf/CfRocksDb.java} (58%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 61f7abb297..c45c4c2ef0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -96,8 +96,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { - log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); - states.put(msg.getId().cfId(), msg.getState()); + CalculatedFieldId cfId = msg.getId().cfId(); + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + if (msg.getState() != null) { + states.put(cfId, msg.getState()); + } else { + states.remove(cfId); + } } public void process(EntityInitCalculatedFieldMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java new file mode 100644 index 0000000000..7df20bc139 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { + + @Autowired + private ActorSystemContext actorSystemContext; + + @Override + public final void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateMsgProto stateMsg = toProto(stateId, state); + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateMsg.getSerializedSize() <= maxStateSizeInKBytes) { + doPersist(stateId, stateMsg, callback); + } + } + + protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback); + + @Override + public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doRemove(stateId, callback); + } + + protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + protected void processRestoredState(CalculatedFieldStateMsgProto stateMsg) { + CalculatedFieldEntityCtxId stateId = fromProto(stateMsg.getId()); + CalculatedFieldState state = stateMsg.hasState() ? fromProto(stateMsg.getState()) : null; + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(stateId, state)); + } + + protected CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + protected CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + protected CalculatedFieldStateMsgProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + var stateProto = CalculatedFieldStateProto.newBuilder() + .setType(state.getType().name()); + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + stateProto.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + stateProto.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + return CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .setState(stateProto) + .build(); + } + + protected TransportProtos.SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + TransportProtos.SingleValueArgumentProto.Builder builder = TransportProtos.SingleValueArgumentProto.newBuilder() + .setArgName(argName); + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + return builder.build(); + } + + protected TransportProtos.TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TransportProtos.TsValueListProto.Builder builder = TransportProtos.TsValueListProto.newBuilder().setKey(argName); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } + return builder.build(); + } + + protected CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + protected SingleValueArgumentEntry fromSingleValueArgumentProto(TransportProtos.SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } + TransportProtos.TsValueProto tsValueProto = proto.getValue(); + long ts = tsValueProto.getTs(); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); + } + + protected TsRollingArgumentEntry fromRollingArgumentProto(TransportProtos.TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } + TreeMap tsRecords = new TreeMap<>(); + proto.getTsValueList().forEach(tsValueProto -> { + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); + tsRecords.put(tsValueProto.getTs(), kvEntry); + }); + return new TsRollingArgumentEntry(tsRecords); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index 37211c66c5..05f9dab36b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -16,14 +16,19 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import java.util.Set; + public interface CalculatedFieldStateService { void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + void restore(Set partitions); + } diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java rename to application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index 9c3c02f472..cc63d757a3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -13,40 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.cf; import jakarta.annotation.PreDestroy; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; +import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.server.utils.TbRocksDb; @Component -public class RocksDBConfig { +public class CfRocksDb extends TbRocksDb { - @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") - private String dbPath; - private RocksDB db; - - static { - RocksDB.loadLibrary(); - } - - public RocksDB getDb() throws RocksDBException { - if (db == null) { - Options options = new Options().setCreateIfMissing(true); - db = RocksDB.open(options, dbPath); - } - return db; + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { + super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); } @PreDestroy + @Override public void close() { - if (db != null) { - db.close(); - db = null; - } + super.close(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 6b47053f2d..f2c6976cbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -70,6 +70,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -91,7 +92,6 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @TbRuleEngineComponent @Service @@ -188,7 +188,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java deleted file mode 100644 index 7181cc43ed..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf; - -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.utils.RocksDBConfig; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -@Service -@Slf4j -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -public class RocksDBService { - - private final RocksDB db; - private final WriteOptions writeOptions; - - public RocksDBService(RocksDBConfig config) throws RocksDBException { - this.db = config.getDb(); - this.writeOptions = new WriteOptions().setSync(true); - } - - public void put(String key, String value) { - try { - db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { - try { - db.put(writeOptions, key.toByteArray(), value.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void delete(CalculatedFieldEntityCtxIdProto key) { - try { - db.delete(writeOptions, key.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - - public String get(String key) { - try { - byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); - return value != null ? new String(value, StandardCharsets.UTF_8) : null; - } catch (RocksDBException e) { - log.error("Failed to retrieve data from RocksDB", e); - return null; - } - } - - public Map getAll() { - Map results = new HashMap<>(); - try (RocksIterator iterator = db.newIterator()) { - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - try { - CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); - CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); - results.put(key, value); - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); - } - } - } - return results; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 9a56d35ac4..b2bb2287fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -21,8 +21,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -30,7 +30,6 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -49,7 +48,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - myPartitions = event.getCalculatedFieldsPartitions().stream() + myPartitions = event.getCfPartitions().stream() .filter(TopicPartitionInfo::isMyPartition) .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); //Naive approach that need to be improved. @@ -58,7 +57,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) .add(profileId, entityId, partition, tpi.isMyPartition()); @@ -66,7 +65,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); //TODO: make this method atomic; @@ -87,7 +86,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); return tpi.getPartition().orElse(UNKNOWN); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 6ce0a11ade..a09a5cd035 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -20,4 +20,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { + + public String toKey() { + return cfId + "_" + entityId; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index a1999d438c..152b1affd9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -15,31 +15,140 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.queue.DefaultTbCalculatedFieldConsumerService.CalculatedFieldQueueConfig; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") -public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final TbRuleEngineQueueFactory queueFactory; + private final PartitionService partitionService; + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> stateConsumer; + private TbKafkaProducerTemplate> stateProducer; + + protected ExecutorService consumersExecutor; + protected ExecutorService mgmtExecutor; + protected ScheduledExecutorService scheduler; - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { + private final AtomicInteger counter = new AtomicInteger(); + + @PostConstruct + private void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("cf-state-consumer")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(Math.max(Runtime.getRuntime().availableProcessors(), 4), "cf-state-mgmt"); + this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("cf-state-consumer-scheduler"); + + this.stateConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(QueueKey.CF_STATES) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg msg : msgs) { + try { + processRestoredState(msg.getValue()); + } catch (Throwable t) { + log.error("Failed to process state message: {}", msg, t); + } + + int processedMsgCount = counter.incrementAndGet(); + if (processedMsgCount % 10000 == 0) { + log.info("Processed {} calculated field state msgs", processedMsgCount); + } + } + }) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - callback.onSuccess(); + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); + stateProducer.send(tpi, stateId.toKey(), new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto), new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (callback != null) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - callback.onSuccess(); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doPersist(stateId, CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .build(), callback); + } + + @Override + public void restore(Set partitions) { + partitions = partitions.stream().map(tpi -> tpi.newByTopic(partitionService.getTopic(QueueKey.CF_STATES))).collect(Collectors.toSet()); + log.info("Restoring calculated field states for partitions: {}", partitions.stream().map(TopicPartitionInfo::getFullTopicName).toList()); + long startTs = System.currentTimeMillis(); + counter.set(0); + + stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update + stateConsumer.awaitStop(0);// consumers should stop on their own because stopWhenRead is true, we just need to wait + + log.info("Restored {} calculated field states in {} ms", counter.get(), System.currentTimeMillis() - startTs); + } + + @PreDestroy + private void preDestroy() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + stateProducer.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 3374eb2660..eb7f4818d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -15,174 +15,57 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.CfRocksDb; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import java.util.Set; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; -public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { - private final ActorSystemContext actorSystemContext; - private final RocksDBService rocksDBService; + private final CfRocksDb cfRocksDb; - public Map restoreStates() { - return rocksDBService.getAll().entrySet().stream() - .collect(Collectors.toMap( - entry -> fromProto(entry.getKey()), - entry -> fromProto(entry.getValue()) - )); - } - - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } + private Set partitions; @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - CalculatedFieldStateProto stateProto = toProto(stateId, state); - long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { - rocksDBService.put(toProto(stateId), stateProto); - } + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); callback.onSuccess(); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(toProto(ctxId)); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + cfRocksDb.delete(stateId.toKey()); callback.onSuccess(); } - private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return CalculatedFieldEntityCtxIdProto.newBuilder() - .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { - TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); - } - - private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { - CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() - .setId(toProto(stateId)) - .setType(state.getType().name()); - - state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { - builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); - } - }); - - return builder.build(); - } - - private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { - SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName); - - if (entry != SingleValueArgumentEntry.EMPTY) { - builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); - } - - Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); - - return builder.build(); - } - - private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { - TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - - if (entry != TsRollingArgumentEntry.EMPTY) { - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); - } - - return builder.build(); - } - - private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { - if (StringUtils.isEmpty(proto.getType())) { - return null; - } - - CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); - - CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); - }; - - proto.getSingleValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - - if (CalculatedFieldType.SCRIPT.equals(type)) { - proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); - } - - return state; - } - - private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { - if (!proto.hasValue()) { - return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + @Override + public void restore(Set partitions) { + if (this.partitions == null) { + this.partitions = partitions; + } else { + return; } - TsValueProto tsValueProto = proto.getValue(); - long ts = tsValueProto.getTs(); - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); - return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); - } - private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { - if (proto.getTsValueCount() <= 0) { - return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; - } - TreeMap tsRecords = new TreeMap<>(); - proto.getTsValueList().forEach(tsValueProto -> { - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); - tsRecords.put(tsValueProto.getTs(), kvEntry); + cfRocksDb.forEach((key, value) -> { + try { + processRestoredState(CalculatedFieldStateMsgProto.parseFrom(value)); + } catch (InvalidProtocolBufferException e) { + log.error("[{}] Failed to process restored state", key, e); + } }); - return new TsRollingArgumentEntry(tsRecords); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index f3e976084b..82ee4de7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,11 +21,11 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -52,6 +52,7 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -65,6 +66,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,10 +89,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private int poolSize; private final TbRuleEngineQueueFactory queueFactory; + private final CalculatedFieldStateService stateService; private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; - private volatile ListeningExecutorService calculatedFieldsExecutor; + private ListeningExecutorService calculatedFieldsExecutor; + private ExecutorService repartitionExecutor; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -100,19 +105,22 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldCache calculatedFieldCache) { + CalculatedFieldCache calculatedFieldCache, + CalculatedFieldStateService stateService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; + this.stateService = stateService; } @PostConstruct public void init() { super.init("tb-cf"); this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + this.repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-cf-repartition")); this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) + .queueKey(QueueKey.CF) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) @@ -137,20 +145,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - var partitions = event.getCalculatedFieldsPartitions(); - log.info("Subscribing to partitions: {}", partitions); - // TODO: @vklimov - before update of the main consumer, we should read the state topics and use - // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. - // Cleanup entities that do not belong to current partition; - mainConsumer.update(event.getCalculatedFieldsPartitions()); - // Cleanup old entities after corresponding consumers are stopped. - // Any periodic tasks need to check that the entity is still managed by the current server before processing. - actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + var partitions = event.getCfPartitions(); + repartitionExecutor.submit(() -> { + try { + stateService.restore(partitions); + mainConsumer.update(partitions); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } catch (Throwable t) { + log.error("Failed to process partition change event: {}", event, t); + } + }); } private boolean[] partitionsToBooleanIndexArray(Set partitions) { boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; - for(var tpi : partitions) { + for (var tpi : partitions) { tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); } return myPartitions; @@ -193,9 +204,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } - if (log.isDebugEnabled()) { - ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); - } +// if (log.isDebugEnabled()) { +// ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); +// } + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); // TODO: replace with commented above after testing ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 48d26895b9..920a7563dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -94,6 +94,7 @@ import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; @@ -111,7 +112,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.thingsboard.server.common.util.ProtoUtils.toProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -358,7 +358,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @@ -371,7 +371,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); toRuleEngineNfs.incrementAndGet(); } @@ -792,8 +792,7 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> - pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 5e07e33a9e..c42f7c490b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -63,8 +64,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @Service @TbRuleEngineComponent @Slf4j @@ -109,7 +108,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { - if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { return; } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java index 6eb5c94c9b..5fa69695d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java @@ -182,7 +182,7 @@ public class MainQueueConsumerManager partitions) { + public void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -226,7 +226,9 @@ public class MainQueueConsumerManager msgs, TbQueueConsumer consumer, C config) throws Exception { + log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, config); + log.trace("Processed {} messages", msgs.size()); } public void stop() { @@ -236,8 +238,12 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 5e672eb5c6..0b4e7c02d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -70,13 +70,21 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(long timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index b9fe54deec..56215b82ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,6 +106,7 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { + systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java new file mode 100644 index 0000000000..fa56e0ec1d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.BiConsumer; + +public class TbRocksDb { + + protected final String path; + private final WriteOptions writeOptions; + protected final RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { + this.path = path; + this.writeOptions = writeOptions; + Files.createDirectories(Path.of(path).getParent()); + this.db = RocksDB.open(dbOptions, path); + } + + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 197f05a5d5..e2570e9137 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,10 +426,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -rocksdb: - # Rocksdb path - db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" - # Actor system parameters actors: system: @@ -1629,6 +1625,8 @@ queue: edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Calculated Field topics calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field State topics + calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1763,6 +1761,8 @@ queue: consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" + # RocksDB path for storing CF states + rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 77a7c4a781..c84cae40dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -146,5 +146,6 @@ public class DataConstants { public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; public static final String CF_QUEUE_NAME = "CalculatedFields"; + public static final String CF_STATES_QUEUE_NAME = "CalculatedFieldStates"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index e89f3c1ad8..7f9a611c98 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -75,4 +75,16 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static boolean isOneOf(V value, V... others) { + if (value == null) { + return false; + } + for (V other : others) { + if (value.equals(other)) { + return true; + } + } + return false; + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4a2f3710ae..992c6ac42f 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -814,12 +814,15 @@ message SingleValueArgumentProto { int64 version = 3; } -message CalculatedFieldStateProto { +message CalculatedFieldStateMsgProto { CalculatedFieldEntityCtxIdProto id = 1; - // int32 version = 2; - string type = 3; - repeated SingleValueArgumentProto singleValueArguments = 4; - repeated TsValueListProto rollingValueArguments = 5; + CalculatedFieldStateProto state = 2; +} + +message CalculatedFieldStateProto { + string type = 1; + repeated SingleValueArgumentProto singleValueArguments = 2; + repeated TsValueListProto rollingValueArguments = 3; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 9513565ca1..6a5b86cf54 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -120,9 +120,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (record != null) { result.add(decode(record)); } - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - throw new RuntimeException("Failed to decode record: ", e); + } catch (Exception e) { + log.error("Failed to decode record {}", record, e); + throw new RuntimeException("Failed to decode record " + record, e); } }); return result; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index f2f1ccd19c..249dfa859f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,7 +51,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Service @Slf4j @@ -78,8 +79,6 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; - public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -120,8 +119,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); - partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); + partitionSizesMap.put(QueueKey.CF, cfPartitions); + partitionTopicsMap.put(QueueKey.CF, cfEventTopic); + partitionSizesMap.put(QueueKey.CF_STATES, cfPartitions); + partitionTopicsMap.put(QueueKey.CF_STATES, cfStateTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); @@ -148,6 +149,11 @@ public class HashPartitionService implements PartitionService { return myPartitions.get(queueKey); } + @Override + public String getTopic(QueueKey queueKey) { + return partitionTopicsMap.get(queueKey); + } + private void doInitRuleEnginePartitions() { List queueRoutingInfoList = getQueueRoutingInfos(); queueRoutingInfoList.forEach(queue -> { @@ -222,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -402,6 +408,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -422,19 +429,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -466,13 +470,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -480,6 +486,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return Collections.emptySet(); + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -508,7 +523,6 @@ public class HashPartitionService implements PartitionService { return result; } - @Override public int resolvePartitionIndex(UUID entityId, int partitions) { int hash = hash(entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b0e1229f97..5bbff7663c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -45,6 +45,8 @@ public interface PartitionService { List getMyPartitions(QueueKey queueKey); + String getTopic(QueueKey queueKey); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index b991e4614c..3f8926ad18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.CF_STATES_QUEUE_NAME; + @Data @AllArgsConstructor public class QueueKey { @@ -32,6 +35,9 @@ public class QueueKey { private final String queueName; private final TenantId tenantId; + public static final QueueKey CF = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + public static final QueueKey CF_STATES = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_STATES_QUEUE_NAME); + public QueueKey(ServiceType type, Queue queue) { this.type = type; this.queueName = queue.getName(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 57a4941981..a16d33c884 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -55,8 +53,8 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } - public Set getCalculatedFieldsPartitions() { - return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); + public Set getCfPartitions() { + return partitionsMap.getOrDefault(QueueKey.CF, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index a2edc35d94..fe2c8b8a90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -23,12 +23,19 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; public class KafkaTbQueueMsg implements TbQueueMsg { + + private static final int UUID_LENGTH = 36; + private final UUID key; private final TbQueueMsgHeaders headers; private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + if (record.key().length() == UUID_LENGTH) { + this.key = UUID.fromString(record.key()); + } else { + this.key = UUID.randomUUID(); + } TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index ef79834735..1d39a3bd0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; @@ -29,9 +31,12 @@ import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +51,15 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached initial end offsets + private Map endOffsets; // needed if stopWhenRead is true + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +77,35 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(topicNames, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } + if (stopWhenRead) { + endOffsets = consumer.endOffsets(partitions).entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + }); + } else { + consumer.subscribe(topicNames); + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -92,13 +124,32 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); - return recordList; + recordList = new ArrayList<>(256); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead && endOffsets != null) { + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + } + if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { + log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + stop(); } + return recordList; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 3c9b85e925..de4fc7049d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -97,9 +97,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; List
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); @@ -116,7 +119,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); + log.warn("Producer template failure", exception); } } }); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index cdd0add38b..7213cf34b8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -54,6 +54,8 @@ public class TbKafkaTopicConfigs { private String housekeeperReprocessingProperties; @Value("${queue.kafka.topic-properties.calculated-field:}") private String calculatedFieldProperties; + @Value("${queue.kafka.topic-properties.calculated-field-state:}") + private String calculatedFieldStateProperties; @Getter private Map coreConfigs; @@ -83,6 +85,8 @@ public class TbKafkaTopicConfigs { private Map edgeEventConfigs; @Getter private Map calculatedFieldConfigs; + @Getter + private Map calculatedFieldStateConfigs; @PostConstruct private void init() { @@ -102,6 +106,7 @@ public class TbKafkaTopicConfigs { edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); + calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index c26e2d15c9..8e36ae4962 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -159,12 +160,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { + public TbQueueConsumer> createCalculatedFieldStateConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { + public TbQueueProducer> createCalculatedFieldStateProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 01e174023c..99a50b2f5a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -100,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -142,6 +143,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -546,26 +548,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 46b35f9acd..4fbf5c3e45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -87,6 +87,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -119,6 +120,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -338,26 +340,28 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index d3c1d09399..5f0af72955 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -126,8 +126,8 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); - TbQueueConsumer> createCalculatedFieldStateConsumer(); + TbQueueConsumer> createCalculatedFieldStateConsumer(); - TbQueueProducer> createCalculatedFieldStateProducer(); + TbQueueProducer> createCalculatedFieldStateProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index dea6d9ed9e..a24f0663b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; - int CF_STATE_RESTORE_SERVICE = 12; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From eae72065ee2dab301efc274e40154ea67302d83d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 13:59:28 +0200 Subject: [PATCH 183/281] fixed integration tests and added debug events creation --- ...CalculatedFieldEntityMessageProcessor.java | 55 ++-- .../CalculatedFieldStateException.java | 2 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 22 +- .../cf/CalculatedFieldIntegrationTest.java | 284 ++++++++++-------- 4 files changed, 198 insertions(+), 165 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index bfed515eb5..66b32dfbfe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -197,12 +197,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = getOrInitState(ctx); justRestored = true; } - if (state.updateState(newArgValues) || justRestored) { - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); - } else { - callback.onSuccess(CALLBACKS_PER_CF); + try { + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } @@ -212,37 +218,38 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - states.put(ctx.getCfId(), state); + try { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); + } + throw new RuntimeException(e); + } } return state; } @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - if (state.isReady() && ctx.isInitialized()) { - try { + try { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); } - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); - } + } else { + callback.onSuccess(); // State was updated but no calculation performed; } - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - try { cfStateService.persistState(ctxId, state, callback); } catch (Exception e) { if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java index 6248ac1536..50ab512cb9 100644 --- a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.exception; -public class CalculatedFieldStateException extends Exception { +public class CalculatedFieldStateException extends RuntimeException { public CalculatedFieldStateException(String message) { super(message); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 0834b5583e..cdad1145e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,6 +25,7 @@ import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.exception.CalculatedFieldStateException; import java.util.ArrayList; import java.util.List; @@ -85,7 +86,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean updateEntry(ArgumentEntry entry) { + public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { updateTsRollingEntry(tsRollingEntry); } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { @@ -107,15 +108,18 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private void addTsRecord(Long ts, KvEntry value) { - switch (value.getDataType()) { - case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); - case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); - case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); - case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - //TODO: try catch + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + cleanupExpiredRecords(); + } catch (Exception e) { + throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); } - cleanupExpiredRecords(); } private void addTsRecord(Long ts, double value) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 91aad7b1cd..f544f463fa 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -40,8 +40,10 @@ import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; @DaoSqlTest public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { @@ -84,20 +86,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -106,32 +110,35 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedOutput.setName("temperatureF"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF argument -> perform calculation with new argument Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); } @Test @@ -164,20 +171,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> state is not ready -> no calculation performed CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> perform calculation doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -211,20 +220,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation with default value CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -275,93 +286,100 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF and perform initial calculation doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - // result of asset 1 - ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); - // result of asset 2 - ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); - Thread.sleep(300); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 1 telemetry -> recalculate state only for asset 1 doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 (no changes) - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 2 telemetry -> recalculate state only for asset 2 doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); - Thread.sleep(300); - - // result of asset 1 (no changes) - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); // add new entity to profile -> calculate state for new entity Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); - Thread.sleep(300); - - // result of asset 3 - ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + Asset finalAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); - // result of asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); // update profile for asset 3 -> delete state for asset 3 AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); @@ -371,22 +389,24 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // update device telemetry -> recalculate state for asset 1 and asset 2 doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); - - // no changes for asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + Asset updatedAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); } @Test @@ -421,20 +441,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> ctx is not initialized -> no calculation perform CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> ctx is not initialized -> no calculation perform doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { From f342bd989ef891be8a56e8ea35ba2ca4c0e024e1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 15:31:09 +0200 Subject: [PATCH 184/281] Minor improvements --- .../auth/rest/RestAuthenticationProvider.java | 1 - .../queue/discovery/HashPartitionService.java | 30 ++++++------------- .../server/queue/kafka/KafkaTbQueueMsg.java | 2 +- .../server/queue/util/AfterStartUp.java | 8 ++--- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 56215b82ac..b9fe54deec 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,7 +106,6 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { - systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 249dfa859f..751e235adc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -228,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); } } @@ -408,7 +408,6 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); - Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -429,16 +428,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); - oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + changedPartitionsMap.put(queueKey, tpiList); } }); if (!changedPartitionsMap.isEmpty()) { changedPartitionsMap.entrySet().stream() .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) - .forEach((serviceType, partitionsMap) -> { - publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); - }); + .forEach(this::publishPartitionChangeEvent); } if (currentOtherServices == null) { @@ -470,15 +469,13 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, - Map> newPartitions, - Map> oldPartitions) { - log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { + log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -486,15 +483,6 @@ public class HashPartitionService implements PartitionService { } } - private Set toTpiList(QueueKey queueKey, List partitions) { - if (partitions == null) { - return Collections.emptySet(); - } - return partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - } - @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index fe2c8b8a90..302f36ff1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -31,7 +31,7 @@ public class KafkaTbQueueMsg implements TbQueueMsg { private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - if (record.key().length() == UUID_LENGTH) { + if (record.key().length() <= UUID_LENGTH) { this.key = UUID.fromString(record.key()); } else { this.key = UUID.randomUUID(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index a24f0663b5..6943fa6753 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From fb0b80a35321710f18a496e46f58f2f8a02699e8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:02:46 +0200 Subject: [PATCH 185/281] Calculated field adjustments --- .../calculated-fields-table-config.ts | 2 +- .../calculated-field-dialog.component.html | 24 ++++++++++++------- .../entity/entities-table.component.ts | 2 +- .../components/event/event-table-config.ts | 13 +++++----- .../entity/entities-table-config.models.ts | 6 ++++- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ce47292397..f1e549d75f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -280,7 +280,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { if (openCalculatedFieldEdit) { - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) } }), ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 6ff0977178..8bd3b5db3f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -100,12 +100,22 @@ [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" helpId="calculated-field/expression_fn" - /> + > + +

@@ -122,18 +132,16 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute) { + @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} - @if (data.entityId.entityType === EntityType.DEVICE) { - - {{ 'calculated-fields.shared-attributes' | translate }} - - } + + {{ 'calculated-fields.shared-attributes' | translate }} + } diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 4f6f34fcce..e472ad5389 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -689,7 +689,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } cellTooltip(entity: BaseData, column: EntityColumn>, row: number) { - if (column instanceof EntityTableColumn) { + if (column instanceof EntityTableColumn || column instanceof EntityLinkTableColumn) { const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellTooltipCache[index]; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index b32c5f0050..a2ccaa05da 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -18,6 +18,7 @@ import { CellActionDescriptorType, DateEntityTableColumn, EntityActionTableColumn, + EntityLinkTableColumn, EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -29,7 +30,7 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityId } from '@shared/models/id/entity-id'; import { EventService } from '@app/core/http/event.service'; import { EventTableHeaderComponent } from '@home/components/event/event-table-header.component'; -import { EntityTypeResource } from '@shared/models/entity-type.models'; +import { EntityType, EntityTypeResource } from '@shared/models/entity-type.models'; import { fromEvent, Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { Direction } from '@shared/models/page/sort-order'; @@ -39,7 +40,7 @@ import { EventContentDialogComponent, EventContentDialogData } from '@home/components/event/event-content-dialog.component'; -import { isEqual, sortObjectKeys } from '@core/utils'; +import { getEntityDetailsPageURL, isEqual, sortObjectKeys } from '@core/utils'; import { DAY, historyInterval, MINUTE } from '@shared/models/time/time.models'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ChangeDetectorRef, EventEmitter, Injector, StaticProvider, ViewContainerRef } from '@angular/core'; @@ -359,13 +360,13 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '100px', + new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, - false, + (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), icon: 'content_paste', @@ -384,7 +385,7 @@ export class EventTableConfig extends EntityTableConfig { () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, + (entity) => entity.body.msgId, false, { name: this.translate.instant('event.copy-message-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index dbdcdcd61e..6c70a11326 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,7 +121,11 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public sortable: boolean = true) { + public cellStyleFunction: CellStyleFunction = () => ({}), + public sortable: boolean = true, + public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), + public cellTooltipFunction: CellTooltipFunction = () => undefined, + public actionCell: CellActionDescriptor = null) { super('link', key, title, width, sortable); } } From 5aacd04e151bb7184dc024c0a6e79c47998b924e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:11:11 +0200 Subject: [PATCH 186/281] Moved calculated fields in device profile tabs --- .../dialog/calculated-field-dialog.component.html | 3 ++- .../device-profile/device-profile-tabs.component.html | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 8bd3b5db3f..2f707c0a2f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -132,7 +132,8 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { + @if (outputFormGroup.get('type').value === OutputType.Attribute + && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { {{ 'calculated-fields.attribute-scope' | translate }} diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index 37cfafa9aa..9a3c34c070 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -40,6 +40,10 @@
+ + + - - From 734064d8b1cc12fb4a222d9feae68b09b77b5816 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:44:04 +0200 Subject: [PATCH 187/281] Switched link column config order --- .../src/app/modules/home/components/event/event-table-config.ts | 2 +- .../modules/home/models/entity/entities-table-config.models.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a2ccaa05da..1e9717d5a8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -363,9 +363,9 @@ export class EventTableConfig extends EntityTableConfig { new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), - () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), + () => ({padding: '0 12px 0 0'}), (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index 6c70a11326..49583e90dc 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,8 +121,8 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public cellStyleFunction: CellStyleFunction = () => ({}), public sortable: boolean = true, + public cellStyleFunction: CellStyleFunction = () => ({}), public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), public cellTooltipFunction: CellTooltipFunction = () => undefined, public actionCell: CellActionDescriptor = null) { From 53f3da153d03828174a1302ef0c68c451562b8b8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 15:28:28 +0200 Subject: [PATCH 188/281] Added debug events action from calculated field table --- .../calculated-fields/calculated-fields-table-config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f1e549d75f..e7ad697262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -118,6 +118,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, onAction: (event$, entity) => this.exportCalculatedField(event$, entity), }, + { + name: this.translate.instant('entity-view.events'), + icon: 'history', + isEnabled: () => true, + onAction: (_, entity) => this.openDebugDialog(entity), + }, { name: '', nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), From 7911384f375f8e823afd51974a53315598673aa4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 14 Feb 2025 16:42:55 +0200 Subject: [PATCH 189/281] fixed tests, added error without stacktrace and param to system params --- .../server/actors/ActorSystemContext.java | 74 +++++++++++-------- ...alculatedFieldManagerMessageProcessor.java | 14 ++-- .../controller/SystemInfoController.java | 11 ++- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../src/main/resources/thingsboard.yml | 6 ++ .../state/ScriptCalculatedFieldStateTest.java | 55 ++------------ .../state/SimpleCalculatedFieldStateTest.java | 10 +-- .../state/SingleValueArgumentEntryTest.java | 7 +- .../ctx/state/TsRollingArgumentEntryTest.java | 60 +++++++++++---- .../server/common/data/SystemParams.java | 1 + .../data/event/CalculatedFieldDebugEvent.java | 1 - 11 files changed, 125 insertions(+), 116 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b8344c6c0a..20f61115e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -43,7 +43,6 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -624,6 +623,14 @@ public class ActorSystemContext { @Getter private String debugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean calculatedFieldsDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String calculatedFieldsDebugPerTenantLimitsConfiguration; + @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -810,39 +817,42 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) { - throw new TbRateLimitsException(EntityType.TENANT); - } - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(calculatedFieldId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString( - arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) - )); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); + if (calculatedFieldsDebugPerTenantEnabled) { + if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!"); } + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(error.getMessage()); + } - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 898e339426..c3aa378524 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -262,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -286,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { - try { - newCfCtx.init(); - } catch (Exception e) { - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } initCf(newCfCtx, callback, stateChanges); } else { callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 8b958df68b..d630c5a22a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -80,6 +80,12 @@ public class SystemInfoController extends BaseController { @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") private String ruleChainDebugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + @Autowired(required = false) private BuildProperties buildProperties; @@ -155,6 +161,9 @@ public class SystemInfoController extends BaseController { if (ruleChainDebugPerTenantLimitsEnabled) { systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); } + if (calculatedFieldDebugPerTenantLimitsEnabled) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration); + } } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index cdad1145e8..1ecf38746e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -43,9 +43,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); this.limit = limit; this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..35e8c0e970 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 30ad7a6507..0227365ed7 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -57,7 +57,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -127,52 +127,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4.0);// will not be used for calculation - values.put(ts - 45000, 2.0);// will not be used for calculation - values.put(ts - 20, 0.0); - - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000.0);// will not be used - values.put(ts - 18, 0.0); - values.put(ts - 16, 0.0); - values.put(ts - 14, 0.0); - values.put(ts - 12, 0.0); - values.put(ts - 10, 0.0); - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); + assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)); } @Test @@ -189,7 +144,7 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenEmptyEntryPresents() { -// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); assertThat(state.isReady()).isFalse(); } @@ -235,7 +190,7 @@ public class ScriptCalculatedFieldStateTest { config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); - config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d87e690478..c451ffcbfc 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest { "key2", key2ArgEntry )); -// Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); -// assertThatThrownBy(() -> state.updateState(newArgs)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @Test @@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest { "key1", key1ArgEntry, "key2", key2ArgEntry )); -// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + state.getArguments().put("key3", new SingleValueArgumentEntry()); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 57e4655d95..2514b9f34e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class SingleValueArgumentEntryTest { @@ -39,9 +40,9 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { -// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index e4d5a59d03..75aca61825 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest { values.put(ts - 30, 12.0); values.put(ts - 20, 17.0); -// entry = new TsRollingArgumentEntry(values); + entry = new TsRollingArgumentEntry(5, 30000L, values); } @Test @@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest { assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); } - @Test - void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); - - assertThat(entry.updateEntry(newEntry)).isFalse(); - } - @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16.0); values.put(ts - 10, 7.0); values.put(ts - 5, 1.0); newEntry.setTsRecords(values); @@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, new DoubleDataEntry("key", 10.0), - ts - 30, new DoubleDataEntry("key", 12.0), - ts - 20, new DoubleDataEntry("key", 17.0), - ts - 10, new DoubleDataEntry("key", 7.0), - ts - 5, new DoubleDataEntry("key", 1.0) + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 )); } @@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest { assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + .hasMessage("Time series rolling arguments supports only numeric values."); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index abe1932327..53f39977c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,5 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java index e0599db358..9b64dd0d40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event { return eventInfo; } - } From 4bd0833eed0df0c694e99abb39ded6d22cc8922e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 18:39:45 +0200 Subject: [PATCH 190/281] Added Calculated field arguments autocomplete --- .../calculated-fields-table-config.ts | 29 +++++-- .../calculated-field-dialog.component.html | 1 + .../calculated-field-dialog.component.ts | 6 ++ ...ed-field-script-test-dialog.component.html | 1 + .../shared/models/calculated-field.models.ts | 79 +++++++++++++++++++ 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index e7ad697262..8f967b68ce 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -36,9 +36,14 @@ import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { + ArgumentType, CalculatedField, + CalculatedFieldArgument, + CalculatedFieldEventArguments, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, + CalculatedFieldRollingValueArgumentAutocomplete, + CalculatedFieldSingleValueArgumentAutocomplete, CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { @@ -48,6 +53,7 @@ import { } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { isObject } from '@core/utils'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +64,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, calculatedField), + action: (calculatedField: CalculatedField) => this.openDebugEventsDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -122,7 +128,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.openDebugDialog(entity), + onAction: (_, entity) => this.openDebugEventsDialog(entity), }, { name: '', @@ -149,7 +155,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(calculatedField) + action: () => this.openDebugEventsDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -211,14 +217,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -267,7 +274,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: CalculatedFieldEventArguments, openCalculatedFieldEdit = true): Observable { const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; @@ -279,6 +286,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig): TbEditorCompleter { + return new TbEditorCompleter(Object.keys(argumentsObj).reduce((acc, key) => { + acc[key] = argumentsObj[key].refEntityKey.type === ArgumentType.Rolling + ? CalculatedFieldRollingValueArgumentAutocomplete + : CalculatedFieldSingleValueArgumentAutocomplete; + return acc; + }, {})) + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 2f707c0a2f..f1e6905aab 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,6 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" + [editorCompleter]="argumentsEditorCompleter$ | async" helpId="calculated-field/expression_fn" >